diff --git a/ast/validate.go b/ast/validate.go index 2f8e377ed..b33cc70dd 100644 --- a/ast/validate.go +++ b/ast/validate.go @@ -5,8 +5,27 @@ import ( "fmt" "math/big" "reflect" + "strings" ) +// WrapValidationError tags a Validate failure with the Python exception +// type it must surface as. Most validation failures are ValueError; a +// handful (NamedExpr / AnnAssign / TypeAlias targets, Constant type) are +// TypeError and already carry that prefix in their message, so they pass +// straight through. Callers that turn a Validate error into a Python +// exception route it through here instead of hardcoding ValueError. +// +// CPython: Python/ast.c PyErr_SetString(PyExc_TypeError/ValueError, ...) +func WrapValidationError(err error) error { + msg := err.Error() + for _, p := range []string{"TypeError: ", "SystemError: "} { + if strings.HasPrefix(msg, p) { + return errors.New(msg) + } + } + return errors.New("ValueError: " + msg) +} + // Validate is the gopy port of _PyAST_Validate. It walks mod and // returns nil if the tree is well-formed, or an error matching // CPython's ValueError/TypeError text. @@ -141,7 +160,7 @@ func validateStmt(s Stmt) error { case *AnnAssign: if n.Target != nil { if _, ok := n.Target.(*Name); !ok && n.Simple != 0 { - return errors.New("AnnAssign with simple non-Name target") + return errors.New("TypeError: AnnAssign with simple non-Name target") } } if err := validateExpr(n.Target, Store); err != nil { @@ -156,7 +175,7 @@ func validateStmt(s Stmt) error { case *TypeAlias: if n.Name != nil { if _, ok := n.Name.(*Name); !ok { - return errors.New("TypeAlias with non-Name name") + return errors.New("TypeError: TypeAlias with non-Name name") } } if err := validateExpr(n.Name, Store); err != nil { @@ -446,7 +465,7 @@ func validateExpr(e Expr, ctx ExprContext) error { return validateExprs(n.Values, Load, false) case *NamedExpr: if _, ok := n.Target.(*Name); !ok { - return errors.New("NamedExpr target must be a Name") + return errors.New("TypeError: NamedExpr target must be a Name") } return validateExpr(n.Value, Load) case *BinOp: @@ -818,7 +837,7 @@ func validateConstant(v any) error { } return nil } - return fmt.Errorf("got an invalid type in Constant: %s", reflect.TypeOf(v)) + return fmt.Errorf("TypeError: got an invalid type in Constant: %s", reflect.TypeOf(v)) } // EllipsisType is the singleton type used to spell Python's `...` as diff --git a/builtins/ast_bridge.go b/builtins/ast_bridge.go index db8228b9e..871ffd2f2 100644 --- a/builtins/ast_bridge.go +++ b/builtins/ast_bridge.go @@ -9,6 +9,8 @@ package builtins import ( + "math/big" + "github.com/tamnd/gopy/ast" "github.com/tamnd/gopy/imp" "github.com/tamnd/gopy/objects" @@ -788,6 +790,8 @@ func (b *astBridge) convertConstantValue(v any) objects.Object { switch x := v.(type) { case int64: return objects.NewInt(x) + case *big.Int: + return objects.NewIntFromBig(x) case float64: return objects.NewFloat(x) case string: @@ -809,6 +813,16 @@ func (b *astBridge) convertConstantValue(v any) objects.Object { items[i] = b.convertConstantValue(elem) } return objects.NewTuple(items) + case ast.FrozenSet: + items := make([]objects.Object, len(x)) + for i, elem := range x { + items[i] = b.convertConstantValue(elem) + } + fs, err := objects.NewFrozenset(items) + if err != nil { + return objects.None() + } + return fs } return objects.None() } diff --git a/builtins/ast_bridge_reverse.go b/builtins/ast_bridge_reverse.go index a6e4e314e..a7b6dd500 100644 --- a/builtins/ast_bridge_reverse.go +++ b/builtins/ast_bridge_reverse.go @@ -123,7 +123,7 @@ func (r *reverseASTBridge) convertStmt(o objects.Object) ast.Stmt { if !ok { return nil } - pos := r.getPos(inst) + pos := r.getPos(inst, "stmt") switch inst.Type().Name { case "Assign": targets := r.exprList(r.getAttr(inst, "targets")) @@ -137,6 +137,17 @@ func (r *reverseASTBridge) convertStmt(o objects.Object) ast.Stmt { op := r.convertOperator(r.getAttr(inst, "op")) value := r.convertExpr(r.getAttr(inst, "value")) return &ast.AugAssign{Target: target, Op: op, Value: value, Pos: pos} + case "AnnAssign": + target := r.convertExpr(r.getAttr(inst, "target")) + annotation := r.convertExpr(r.getAttr(inst, "annotation")) + value := r.optionalExpr(inst, "value") + return &ast.AnnAssign{ + Target: target, + Annotation: annotation, + Value: value, + Simple: r.getAttrInt(inst, "simple"), + Pos: pos, + } case "Return": val := r.getAttr(inst, "value") var retVal ast.Expr @@ -227,6 +238,11 @@ func (r *reverseASTBridge) convertStmt(o objects.Object) ast.Stmt { subject := r.convertExpr(r.getAttr(inst, "subject")) cases := r.convertMatchCases(r.getAttr(inst, "cases")) return &ast.Match{Subject: subject, Cases: cases, Pos: pos} + case "TypeAlias": + name := r.convertExpr(r.getAttr(inst, "name")) + typeParams := r.convertTypeParams(r.getAttr(inst, "type_params")) + value := r.convertExpr(r.getAttr(inst, "value")) + return &ast.TypeAlias{Name: name, TypeParams: typeParams, Value: value, Pos: pos} } // Unknown statement: emit a Pass so the body remains valid. return &ast.Pass{Pos: pos} @@ -243,7 +259,7 @@ func (r *reverseASTBridge) convertExceptHandlers(o objects.Object) ast.Seq[ast.E if !ok { continue } - pos := r.getPos(inst) + pos := r.getPos(inst, "excepthandler") var typ ast.Expr if t := r.getAttr(inst, "type"); t != nil && t != objects.None() { typ = r.convertExpr(t) @@ -315,7 +331,7 @@ func (r *reverseASTBridge) convertPattern(o objects.Object) ast.Pattern { panic(astRecursionSentinel{}) } defer func() { r.depth++ }() - pos := r.getPos(inst) + pos := r.getPos(inst, "pattern") switch inst.Type().Name { case "MatchValue": value := r.convertExpr(r.getAttr(inst, "value")) @@ -425,6 +441,7 @@ func (r *reverseASTBridge) convertFunctionDef(inst *objects.Instance, pos ast.Po Body: body, DecoratorList: decorators, Returns: returns, + TypeParams: r.convertTypeParams(r.getAttr(inst, "type_params")), Pos: pos, } } @@ -445,6 +462,7 @@ func (r *reverseASTBridge) convertAsyncFunctionDef(inst *objects.Instance, pos a Body: body, DecoratorList: decorators, Returns: returns, + TypeParams: r.convertTypeParams(r.getAttr(inst, "type_params")), Pos: pos, } } @@ -461,10 +479,69 @@ func (r *reverseASTBridge) convertClassDef(inst *objects.Instance, pos ast.Pos) Keywords: keywords, Body: body, DecoratorList: decorators, + TypeParams: r.convertTypeParams(r.getAttr(inst, "type_params")), Pos: pos, } } +// convertTypeParams reverses convertTypeParams in the forward bridge: +// it rebuilds the Go PEP 695 type-parameter nodes (TypeVar / TypeVarTuple +// / ParamSpec) from their _ast instances so a compile()-from-AST request +// carrying generic functions, classes, or type aliases reaches codegen +// (and the validator) with its type_params intact. +// +// CPython: Python/Python-ast.c obj2ast_type_param +func (r *reverseASTBridge) convertTypeParams(o objects.Object) ast.Seq[ast.TypeParam] { + lst, ok := o.(*objects.List) + if !ok { + return nil + } + out := make(ast.Seq[ast.TypeParam], 0, lst.Len()) + for i := 0; i < lst.Len(); i++ { + inst, ok := lst.Item(i).(*objects.Instance) + if !ok { + continue + } + pos := r.getPos(inst, "type_param") + name := r.getAttrString(inst, "name") + switch inst.Type().Name { + case "TypeVar": + var bound ast.Expr + if b := r.getAttr(inst, "bound"); b != nil && b != objects.None() { + bound = r.convertExpr(b) + } + out = append(out, &ast.TypeVar{ + Name: name, + Bound: bound, + DefaultValue: r.optionalExpr(inst, "default_value"), + Pos: pos, + }) + case "TypeVarTuple": + out = append(out, &ast.TypeVarTuple{ + Name: name, + DefaultValue: r.optionalExpr(inst, "default_value"), + Pos: pos, + }) + case "ParamSpec": + out = append(out, &ast.ParamSpec{ + Name: name, + DefaultValue: r.optionalExpr(inst, "default_value"), + Pos: pos, + }) + } + } + return out +} + +// optionalExpr converts attr name to a Go expr, returning nil when the +// attribute is missing or None. +func (r *reverseASTBridge) optionalExpr(inst *objects.Instance, name string) ast.Expr { + if v := r.getAttr(inst, name); v != nil && v != objects.None() { + return r.convertExpr(v) + } + return nil +} + func (r *reverseASTBridge) convertArguments(o objects.Object) *ast.Arguments { inst, ok := o.(*objects.Instance) if !ok { @@ -515,7 +592,7 @@ func (r *reverseASTBridge) convertArg(o objects.Object) *ast.Arg { } a := &ast.Arg{ Arg: r.getAttrString(inst, "arg"), - Pos: r.getPos(inst), + Pos: r.getPos(inst, "arg"), } if ann := r.getAttr(inst, "annotation"); ann != nil && ann != objects.None() { a.Annotation = r.convertExpr(ann) @@ -575,7 +652,7 @@ func (r *reverseASTBridge) convertExpr(o objects.Object) ast.Expr { panic(astRecursionSentinel{}) } defer func() { r.depth++ }() - pos := r.getPos(inst) + pos := r.getPos(inst, "expr") switch inst.Type().Name { case "Name": id := r.getAttrIdentifier(inst, "id") @@ -749,7 +826,7 @@ func (r *reverseASTBridge) convertKeywords(o objects.Object) ast.Seq[*ast.Keywor arg = &s } val := r.convertExpr(r.getAttr(inst, "value")) - out = append(out, &ast.Keyword{Arg: arg, Value: val, Pos: r.getPos(inst)}) + out = append(out, &ast.Keyword{Arg: arg, Value: val, Pos: r.getPos(inst, "keyword")}) } return out } @@ -786,7 +863,7 @@ func (r *reverseASTBridge) convertAliases(o objects.Object) ast.Seq[*ast.Alias] s := r.getAttrString(inst, "asname") asname = &s } - out = append(out, &ast.Alias{Name: name, Asname: asname, Pos: r.getPos(inst)}) + out = append(out, &ast.Alias{Name: name, Asname: asname, Pos: r.getPos(inst, "alias")}) } return out } @@ -966,8 +1043,12 @@ func (r *reverseASTBridge) convertConstantValue(o objects.Object) any { case *objects.Unicode: return v.Value() case *objects.Int: - i64, _ := v.Int64() - return i64 + // Constants that overflow int64 must round-trip as *big.Int, the + // same representation the parser emits, so co_consts compares equal. + if i64, ok := v.Int64(); ok { + return i64 + } + return v.BigInt() case *objects.Float: return v.Float64() case *objects.Complex: @@ -1060,14 +1141,25 @@ func (r *reverseASTBridge) getAttrPresent(inst *objects.Instance, name string) ( // matching CPython's Python/Python-ast.c:11187 obj2ast_stmt. // // CPython: Python/ast.c:1043 validate_stmt LOCATION macro -func (r *reverseASTBridge) getPos(inst *objects.Instance) ast.Pos { +// getPos reads the source-location attributes that every located AST node +// (stmt, expr, excepthandler, pattern, type_param, arg, keyword, alias) +// carries. lineno and col_offset are required: a missing attribute raises +// TypeError("required field \"lineno\" missing from "), matching the +// obj2ast PyObject_GetOptionalAttr == NULL path. end_lineno / end_col_offset +// default to lineno / col_offset when absent or None. +// +// CPython: Python/Python-ast.c:11149 obj2ast_stmt attribute block +func (r *reverseASTBridge) getPos(inst *objects.Instance, kind string) ast.Pos { linenoVal, linenoPresent := r.getAttrPresent(inst, "lineno") if !linenoPresent { - return ast.NoPos + panic(astTypeError{fmt.Sprintf("required field \"lineno\" missing from %s", kind)}) } if linenoVal == objects.None() { panic(astValidationError{"invalid integer value: None"}) } + if _, colPresent := r.getAttrPresent(inst, "col_offset"); !colPresent { + panic(astTypeError{fmt.Sprintf("required field \"col_offset\" missing from %s", kind)}) + } lineno := r.getAttrInt(inst, "lineno") colOffset := r.getAttrInt(inst, "col_offset") diff --git a/builtins/compile.go b/builtins/compile.go index 2388516c0..56e9cf3ad 100644 --- a/builtins/compile.go +++ b/builtins/compile.go @@ -12,6 +12,7 @@ import ( "fmt" "github.com/tamnd/gopy/ast" + "github.com/tamnd/gopy/codecs" "github.com/tamnd/gopy/compile" "github.com/tamnd/gopy/objects" "github.com/tamnd/gopy/parser" @@ -66,7 +67,7 @@ func Compile(args []objects.Object, kwargs map[string]objects.Object) (objects.O if err != nil { return nil, err } - return liftCompileCode(cco), nil + return LiftCompileCode(cco), nil } type compileArgs struct { @@ -121,6 +122,16 @@ func parseCompileArgs(args []objects.Object, kwargs map[string]objects.Object) ( if err != nil { return compileArgs{}, err } + // When the source is an AST object, its top node must match the + // requested mode: exec wants Module, eval Expression, single + // Interactive. PyAST_obj2mod rejects a mismatch before conversion. + // + // CPython: Python/Python-ast.c:18427 PyAst_CheckMode + if astMod != nil { + if err := checkASTMode(astMod, bound[0], mode); err != nil { + return compileArgs{}, err + } + } flags, err := parseCompileFlags(bound[3]) if err != nil { return compileArgs{}, err @@ -184,6 +195,15 @@ func compileFilenameArg(o objects.Object) (string, error) { func compileSourceArg(o objects.Object) (string, []byte, error) { switch v := o.(type) { case *objects.Unicode: + // _Py_SourceAsString encodes a str source through the strict + // utf-8 codec (PyUnicode_AsUTF8AndSize), so a lone surrogate + // raises UnicodeEncodeError here rather than reaching the + // tokenizer as a "Non-UTF-8 code" SyntaxError. + // + // CPython: Python/pythonrun.c:1572 _Py_SourceAsString + if _, _, encErr := codecs.Encode(v.Value(), "utf-8", "strict"); encErr != nil { + return "", nil, encErr + } return v.Value(), nil, nil case *objects.Bytes: b := v.Bytes() @@ -292,6 +312,37 @@ func parseCompileMode(modeStr string) (parser.Mode, error) { return 0, fmt.Errorf("ValueError: compile() mode must be 'exec', 'eval', 'single' or 'func_type'") } +// checkASTMode enforces that an AST source object's top node matches the +// requested compile mode: exec wants Module, eval Expression, single +// Interactive. CPython does this with an isinstance() check against the +// per-mode required type and raises TypeError("expected %s node, got +// %.400s") on a mismatch. +// +// CPython: Python/Python-ast.c:18427 PyAst_CheckMode +func checkASTMode(astMod ast.Mod, src objects.Object, mode parser.Mode) error { + var reqName string + var match bool + switch mode { + case parser.ModeFile: + reqName = "Module" + _, match = astMod.(*ast.Module) + case parser.ModeEval: + reqName = "Expression" + _, match = astMod.(*ast.Expression) + case parser.ModeSingle: + reqName = "Interactive" + _, match = astMod.(*ast.Interactive) + default: + // func_type and any future mode have no Module/Expression/ + // Interactive requirement; leave validation to conversion. + return nil + } + if match { + return nil + } + return fmt.Errorf("TypeError: expected %s node, got %.400s", reqName, src.Type().Name) +} + // PyCF_ONLY_AST and PyCF_OPTIMIZED_AST flag constants. // // CPython: Include/cpython/code.h PyCF_ONLY_AST / PyCF_OPTIMIZED_AST @@ -360,18 +411,17 @@ func parseCompileFlags(o objects.Object) (int, error) { // checkDontInherit accepts dont_inherit for signature parity. gopy has // no surrounding compiler-flags context to inherit, so the value is a -// no-op either way; only the type is validated. +// no-op either way, but the argument is still run through the truth test +// so a misbehaving __bool__ / __len__ surfaces the same error CPython's +// PyObject_IsTrue conversion does. +// +// CPython: Python/clinic/bltinmodule.c.h:341 dont_inherit = PyObject_IsTrue(args[4]) func checkDontInherit(o objects.Object) error { if o == nil { return nil } - if _, ok := o.(*objects.Int); ok { - return nil - } - if _, ok := o.(*objects.Bool); ok { - return nil - } - return fmt.Errorf("TypeError: compile() arg 5 (dont_inherit) must be int or bool") + _, err := objects.IsTruthy(o) + return err } // parseCompileOptimize reads the optional optimize arg. -1 is the @@ -438,7 +488,7 @@ func signedIntArg(o objects.Object, label string) (int, error) { func parseOnlyResult(mod ast.Mod, parsed *compileArgs) (objects.Object, error) { // CPython: Python/bltinmodule.c:843 _PyAST_Validate if err := ast.Validate(mod); err != nil { - return nil, fmt.Errorf("ValueError: %w", err) + return nil, ast.WrapValidationError(err) } // CPython: Python/bltinmodule.c:846 // syntax_check_only = ((flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST) @@ -457,6 +507,20 @@ func parseOnlyResult(mod ast.Mod, parsed *compileArgs) (objects.Object, error) { return astModToObject(mod), nil } +// LiftCompileCode exposes liftCompileCode so the _testinternalcapi +// compiler-pipeline helpers can turn an assembled compile.Code into the +// objects.Code that CPython's _PyCompile_Assemble hands back. It is the +// top-level lift boundary, so it also runs the per-compile constant +// merge that shares equal co_consts / co_linetable / co_filename +// objects across the whole code tree. +// +// CPython: Python/compile.c:1707 const_cache lifetime +func LiftCompileCode(c *compile.Code) *objects.Code { + top := liftCompileCode(c) + objects.InternCodeConstants(top) + return top +} + // liftCompileCode adapts compile.Code into objects.Code. Mirrors the // helper pythonrun keeps for the same purpose; both go away once spec // 1687 retires compile.Code in favor of objects.Code directly. @@ -516,6 +580,17 @@ func liftCompileConst(v any) any { items[i] = liftCompileConst(raw) } return items + case *compile.ConstSlice: + return objects.NewSliceFromConst(x.Start, x.Stop, x.Step) + case ast.FrozenSet: + // A frozenset const may hold tuple elements that codegen left as + // *compile.ConstTuple; lift each one so marshal and the runtime + // see only native values. + items := make(ast.FrozenSet, len(x)) + for i, raw := range x { + items[i] = liftCompileConst(raw) + } + return items } return v } diff --git a/builtins/eval.go b/builtins/eval.go index d6c28351d..781dbfbbf 100644 --- a/builtins/eval.go +++ b/builtins/eval.go @@ -13,6 +13,7 @@ import ( "fmt" "github.com/tamnd/gopy/ast" + "github.com/tamnd/gopy/codecs" "github.com/tamnd/gopy/compile" "github.com/tamnd/gopy/objects" "github.com/tamnd/gopy/parser" @@ -281,11 +282,11 @@ func codeForSource(source objects.Object, fnName string, mode parser.Mode) (*obj } mod = m } - cco, err := compile.Compile(mod, "", 0) + cco, err := compile.Compile(mod, "", -1) if err != nil { return nil, err } - return liftCompileCode(cco), nil + return LiftCompileCode(cco), nil } // sourceAsString is the gopy port of CPython's _Py_SourceAsString. @@ -303,12 +304,34 @@ func sourceAsString(cmd objects.Object, fnName string) (string, bool, error) { isUnicode := false switch v := cmd.(type) { case *objects.Unicode: + // _Py_SourceAsString routes a str through PyUnicode_AsUTF8AndSize, + // the strict utf-8 encoder. A source string carrying a lone + // surrogate (e.g. the console fed "'\ud800'") therefore raises + // UnicodeEncodeError here, before the tokenizer ever runs, rather + // than surfacing as the lexer's "Non-UTF-8 code" SyntaxError. + // + // CPython: Python/pythonrun.c:1572 _Py_SourceAsString + // (PyUnicode_AsUTF8AndSize) + if _, _, encErr := codecs.Encode(v.Value(), "utf-8", "strict"); encErr != nil { + return "", false, encErr + } s = v.Value() isUnicode = true case *objects.Bytes: s = string(v.Bytes()) case *objects.ByteArray: s = string(v.Bytes()) + case *objects.MemoryView: + // _Py_SourceAsString accepts any object exposing the buffer + // protocol; memoryview is the gopy surface for that, so a + // sliced memoryview feeds eval()/exec()/compile() straight + // through here. + // + // CPython: Python/pythonrun.c:1572 _Py_SourceAsString (PyObject_CheckBuffer) + if err := objects.CheckBufferReleased(v); err != nil { + return "", false, err + } + s = string(v.Bytes()) default: return "", false, fmt.Errorf("TypeError: %s() arg 1 must be a string, bytes or code object", fnName) } diff --git a/builtins/init.go b/builtins/init.go index 034997686..2bd651643 100644 --- a/builtins/init.go +++ b/builtins/init.go @@ -74,6 +74,14 @@ func Init(defaultFile io.Writer) (*objects.Dict, error) { if err := setBuiltin(dict, "Ellipsis", objects.Ellipsis()); err != nil { return nil, err } + // __debug__ is True unless the interpreter runs with -O. gopy has no + // optimize flag yet, so it is always True, matching CPython's + // bltinmod_exec which stores Py_True under "__debug__". + // + // CPython: Python/bltinmodule.c:3197 _PyBuiltin_Init + if err := setBuiltin(dict, "__debug__", objects.True()); err != nil { + return nil, err + } for _, t := range typeSingletons() { if err := setBuiltin(dict, t.name, t.t); err != nil { diff --git a/cmd/gopy/main.go b/cmd/gopy/main.go index e784900fc..c48fa42b9 100644 --- a/cmd/gopy/main.go +++ b/cmd/gopy/main.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "runtime/pprof" + "strconv" "strings" "github.com/tamnd/gopy/build" @@ -105,6 +106,7 @@ func run(args []string, stdout, stderr *os.File) int { hasC, hasM bool xOptions []string safePath bool + optimize int ) opts: @@ -130,6 +132,14 @@ opts: break opts case 'X': xOptions = append(xOptions, st.OptArg) + case 'O': + // -O / -OO raise the optimization level. CPython's + // config_parse_cmdline increments optimization_level per 'O'; + // the compiler reads it through _Py_GetConfig() to drop + // asserts (level 1) and docstrings (level 2). + // + // CPython: Python/initconfig.c:2068 config_parse_cmdline ('O') + optimize++ case 'P': // -P sets safe_path: the script directory / cwd / '' is not // prepended to sys.path[0]. @@ -158,6 +168,26 @@ opts: codecs.SetDevMode(true) } + // optimization level: -O / -OO on the command line plus PYTHONOPTIMIZE + // in the environment. CPython folds the environment value in + // (taking the max) and exposes the result as both sys.flags.optimize + // and _Py_GetConfig()->optimization_level; the compiler resolves an + // optimize == -1 request against it. Without -E the env value still + // applies. + // + // CPython: Python/initconfig.c:1700 config_init_optimization_level + if env := os.Getenv("PYTHONOPTIMIZE"); env != "" { + if n, err := strconv.Atoi(env); err == nil && n > optimize { + optimize = n + } else if err != nil && len(env) > optimize { + // A non-numeric PYTHONOPTIMIZE counts its length, matching + // CPython's _PyLong_FromString fallback for the env knob. + optimize = len(env) + } + } + compile.SetConfigOptimize(optimize) + sys.SetOptimize(optimize) + // safe_path: -P, -I, or PYTHONSAFEPATH suppresses prepending the // script directory / cwd / "" to sys.path[0], and is exposed as // sys.flags.safe_path. installPathFinder reads safePathMode to skip diff --git a/compile/assemble_exceptions.go b/compile/assemble_exceptions.go index 75d28efb4..86b637a9d 100644 --- a/compile/assemble_exceptions.go +++ b/compile/assemble_exceptions.go @@ -67,8 +67,11 @@ func AssembleExceptionTable(seq *Sequence) []byte { return assembleExceptionTabl // assembleExceptionTable walks the post-flowgraph instruction stream, // groups runs that share a handler, and emits one varint record per -// run. The byte-offset cursor mirrors how the code stream is packed -// (one codeunit = two bytes, plus EXTENDED_ARG prefixes). +// run. Offsets are in code units (one codeunit per opcode plus its +// EXTENDED_ARG prefixes and inline caches), matching CPython's +// exception-table format: dis.py and the unwinder both read these +// values as instruction offsets and scale by the codeunit size, so +// the serialized table must NOT pre-multiply to bytes. // // CPython: Python/assemble.c:L157 assemble_exception_table func assembleExceptionTable(seq *Sequence) []byte { @@ -79,7 +82,7 @@ func assembleExceptionTable(seq *Sequence) []byte { off := 0 for i := range seq.Instrs { offsets[i] = off - off += instrSize(&seq.Instrs[i]) * 2 + off += instrSize(&seq.Instrs[i]) } totalBytes := off diff --git a/compile/codegen.go b/compile/codegen.go index ef80efd67..9c74dacb8 100644 --- a/compile/codegen.go +++ b/compile/codegen.go @@ -8,6 +8,7 @@ package compile import ( "fmt" + "slices" "github.com/tamnd/gopy/ast" "github.com/tamnd/gopy/future" @@ -193,6 +194,15 @@ type Compiler struct { // enterScope. fblocks []fblock + // disableWarning suppresses compile-time SyntaxWarnings while + // non-zero. The exception-path copy of a finally body is compiled a + // second time (under a FINALLY_END fblock), so without this guard a + // warning inside the finally clause would be emitted twice. Pushing + // the FINALLY_END fblock bumps this counter; popping it restores it. + // + // CPython: Python/compile.c:106 compiler.c_disable_warning + disableWarning int + // interactive marks compile-mode='single' (REPL / doctest). When // set, expression-statements at module nest level emit // CALL_INTRINSIC_1 INTRINSIC_PRINT so each result reaches @@ -202,6 +212,16 @@ type Compiler struct { // CPython: Python/codegen.c codegen_stmt_expr (c->c_interactive && // c->c_nestlevel <= 1 fires PRINT_EXPR) interactive bool + + // saveNestedSeqs makes leaveScope hang each finished scope's + // instruction sequence off its parent unit's Seq.Nested, so a + // caller can walk the recursive pre-flowgraph instruction tree. + // Only the _testinternalcapi.compiler_codegen helper sets it; the + // normal compile path leaves it false (nested scopes flow through + // the const pool as *Unit instead). + // + // CPython: Python/compile.c:103 compiler.c_save_nested_seqs + saveNestedSeqs bool } // NewCompiler builds a fresh driver. Symtable must already be built @@ -241,6 +261,20 @@ func (c *Compiler) Codegen(sc *symtable.Entry, mod ast.Mod) (*Unit, error) { resumeLoc := ast.Pos{Lineno: 0, EndLineno: c.unit().FirstLineno, ColOffset: 0, EndColOffset: 0} c.addOpI(RESUME, resumeAtFuncStart, resumeLoc) + // Module scope plants ANNOTATIONS_PLACEHOLDER right after RESUME. The + // deferred __annotate__ build (stashed via SetAnnotationsCode) is spliced + // in at this marker by cfgFromSequence, so it lands after RESUME and + // before the body's BUILD_SET/STORE __conditional_annotations__ prologue. + // CPython plants it unconditionally and drops it when no stash exists; we + // only plant it when the module carries (always-conditional) annotations, + // which is exactly when a stash is produced. The spliced-out result is + // byte-identical either way. + // + // CPython: Python/codegen.c:659 codegen_enter_scope (COMPILE_SCOPE_MODULE) + if sc.HasConditionalAnnotations { + c.addOp(ANNOTATIONS_PLACEHOLDER, resumeLoc) + } + switch m := mod.(type) { case *ast.Module: if err := c.visitModule(m); err != nil { @@ -426,6 +460,19 @@ func (c *Compiler) enterScope(sc *symtable.Entry) { } sortStrings(cellNames) sortStrings(freeNames) + // __conditional_annotations__ is forced into u_cellvars (after the + // sorted cells) when the scope tracks conditional annotations, even + // where the symtable left the name GLOBAL. That happens at module + // scope: the generated __annotate__ reads the set via LOAD_GLOBAL, so + // the name never goes free and never resolves to CELL, yet the code + // object still needs the cell so MAKE_CELL runs at the prologue. Class + // scopes already resolve it to CELL above (the annotate body uses + // LOAD_DEREF), so this add is a no-op there, matching DictAddObj. + // + // CPython: Python/compile.c:630 compiler_enter_scope (DictAddObj cellvars) + if sc.HasConditionalAnnotations && !slices.Contains(cellNames, "__conditional_annotations__") { + cellNames = append(cellNames, "__conditional_annotations__") + } for _, name := range cellNames { u.CellVars = append(u.CellVars, name) c.cellCache[name] = len(u.CellVars) - 1 @@ -518,11 +565,19 @@ func (c *Compiler) leaveScope() { if len(c.units) == 0 { return } + child := c.units[len(c.units)-1] c.units = c.units[:len(c.units)-1] if len(c.units) > 0 { // scope tracking only matters for the active unit. The // driver re-enters the parent scope explicitly. c.scope = nil + // _PyCompile_ExitScope appends the finished child sequence to + // the parent under c_save_nested_seqs, in scope-exit order. + // + // CPython: Python/compile.c:719 _PyCompile_ExitScope + if c.saveNestedSeqs && child != nil && child.Seq != nil { + c.units[len(c.units)-1].Seq.AddNested(child.Seq) + } } } diff --git a/compile/codegen_addop.go b/compile/codegen_addop.go index 0a28baa7e..24cd56d12 100644 --- a/compile/codegen_addop.go +++ b/compile/codegen_addop.go @@ -193,6 +193,22 @@ func appendConstKey(b *strings.Builder, value any) { appendConstKey(b, item) } b.WriteString(")") + case *ConstSlice: + // A slice constant carries its own bracketing tag so it never + // shares a cache slot with a tuple of the same three values. + b.WriteString("[") + appendConstKey(b, x.Start) + appendConstKey(b, x.Stop) + appendConstKey(b, x.Step) + b.WriteString("]") + case ast.FrozenSet: + // A frozenset constant must never share a cache slot with the + // equivalent tuple, so it carries its own bracketing tag. + b.WriteString("{") + for _, item := range x { + appendConstKey(b, item) + } + b.WriteString("}") default: // Inner code units and any other reference-typed value get a // pointer-identity tag so two distinct objects never collide diff --git a/compile/codegen_annotations.go b/compile/codegen_annotations.go index 9cd643153..c71ec6a64 100644 --- a/compile/codegen_annotations.go +++ b/compile/codegen_annotations.go @@ -253,18 +253,12 @@ func (c *Compiler) emitAnnotateBody(innerScope *symtable.Entry, deferred []defer // Python/compile.c:762 _PyCompile_EndAnnotationSetup func (c *Compiler) stashAnnotationCode(l ast.Pos) error { u := c.unit() - hasCondAnno := c.scope != nil && c.scope.HasConditionalAnnotations - if len(u.DeferredAnnotations) == 0 && !hasCondAnno { + if len(u.DeferredAnnotations) == 0 { return nil } mainSeq := u.Seq annoSeq := &Sequence{} u.Seq = annoSeq - if hasCondAnno { - pool := poolNames - c.addOpI(BUILD_SET, 0, l) - c.addOpName(STORE_NAME, &pool, "__conditional_annotations__", l) - } if err := c.emitDeferredAnnotations(l); err != nil { u.Seq = mainSeq return err @@ -276,8 +270,26 @@ func (c *Compiler) stashAnnotationCode(l ast.Pos) error { return nil } +// emitConditionalAnnotationsPrologue emits the BUILD_SET / STORE_NAME +// pair that seeds __conditional_annotations__ at the top of a module +// body. CPython runs this in _PyCodegen_Module before codegen_body, so +// the name is registered first (names[0]) and the set exists before any +// annotated statement records itself into it. Class bodies emit the +// STORE_DEREF form inline in visitClassBody instead. +// +// CPython: Python/codegen.c:858 _PyCodegen_Module (BUILD_SET/STORE_NAME) +func (c *Compiler) emitConditionalAnnotationsPrologue(l ast.Pos) error { + if c.scope == nil || !c.scope.HasConditionalAnnotations { + return nil + } + pool := poolNames + c.addOpI(BUILD_SET, 0, l) + c.addOpName(STORE_NAME, &pool, "__conditional_annotations__", l) + return nil +} + // emitFunctionAnnotations compiles the __annotate__ function for a -// function definition and leaves it on the outer stack. Returns 0x04 +// function definition and leaves it on the outer stack. Returns 0x10 // (MAKE_FUNCTION_ANNOTATE) when annotations exist, 0 otherwise. // // The AnnotationBlock for the function's args is looked up via @@ -302,7 +314,7 @@ func (c *Compiler) emitFunctionAnnotations(args *ast.Arguments, returns ast.Expr return 0, err } c.emitMakeFunction(closureFlag, l) - return 0x04, nil + return 0x10, nil } // emitFunctionAnnotateBody pushes a fresh Unit for the function's diff --git a/compile/codegen_class.go b/compile/codegen_class.go index 05a6d211b..16121cfb5 100644 --- a/compile/codegen_class.go +++ b/compile/codegen_class.go @@ -57,6 +57,18 @@ func (c *Compiler) visitClassDef(s *ast.ClassDef) error { return c.nameOpStore(s.Name, loc(s)) } +// classFirstLineno returns the source line CPython assigns as a class's +// co_firstlineno: the first decorator's line when the class is +// decorated, otherwise 0 (meaning "leave the symtable-derived line"). +// +// CPython: Python/codegen.c:1629 codegen_class +func classFirstLineno(s *ast.ClassDef) int { + if len(s.DecoratorList) > 0 { + return loc(s.DecoratorList[0]).Lineno + } + return 0 +} + // emitClassBuildCall lays out the LOAD_BUILD_CLASS + body fn + name + // bases + keywords call. Shared between the non-generic path (called // directly from visitClassDef) and the generic path (called inside the @@ -319,6 +331,9 @@ func (c *Compiler) emitGenericClass(s *ast.ClassDef) error { // CPython: Python/compile.c compiler_enter_scope (private param = class_name // for TypeParams, Python/codegen.c:1623 codegen_class) c.unit().Private = s.Name + if ln := classFirstLineno(s); ln > 0 { + c.unit().FirstLineno = ln + } first := c.unit().FirstLineno c.addOpI(RESUME, 0, ast.Pos{Lineno: first, EndLineno: first}) @@ -452,6 +467,16 @@ func (c *Compiler) emitInnerClassCode(innerScope *symtable.Entry, s *ast.ClassDe outerCaches := c.savedCaches() c.enterScope(innerScope) + // A decorated class's co_firstlineno (and __firstlineno__) is the + // line of its first decorator, not the `class` line. enterScope + // derives firstlineno from the symtable entry (the class line), so + // override it here. + // + // CPython: Python/codegen.c:1629 codegen_class (firstlineno = + // decos[0]->lineno when the decorator list is non-empty) + if ln := classFirstLineno(s); ln > 0 { + c.unit().FirstLineno = ln + } // RESUME loc = LOCATION(firstlineno, firstlineno, 0, 0); CPython // drops the def-stmt's columns on the class-entry RESUME. // diff --git a/compile/codegen_comp_inline.go b/compile/codegen_comp_inline.go index d674b8b40..8e6412cdd 100644 --- a/compile/codegen_comp_inline.go +++ b/compile/codegen_comp_inline.go @@ -46,7 +46,8 @@ func (c *Compiler) tweakInlinedComprehensionScopes(comp *symtable.Entry, state * inClassBlock := c.scope.Type == symtable.ClassBlock && u.InInlinedComp == 0 u.InInlinedComp++ - for k, symbol := range comp.Symbols { + for _, k := range comp.OrderedSymbols() { + symbol := comp.Symbols[k] scope := symbol.Scope() outsymbol := c.scope.GetSymbol(k) outsc := outsymbol.Scope() @@ -107,7 +108,8 @@ func (c *Compiler) pushInlinedComprehensionLocals(comp *symtable.Entry, state *i u := c.unit() inClassBlock := c.scope.Type == symtable.ClassBlock && u.InInlinedComp == 0 - for k, symbol := range comp.Symbols { + for _, k := range comp.OrderedSymbols() { + symbol := comp.Symbols[k] scope := symbol.Scope() if (symbol&symtable.DefLocal != 0 && symbol&symtable.DefNonlocal == 0) || inClassBlock { // Push the existing value (which may be NULL if not defined) diff --git a/compile/codegen_control_test.go b/compile/codegen_control_test.go index 66c4cb988..7fa82427d 100644 --- a/compile/codegen_control_test.go +++ b/compile/codegen_control_test.go @@ -43,8 +43,8 @@ func TestIfWithElseEmitsJumpAroundElse(t *testing.T) { "TO_BOOL", "POP_JUMP_IF_FALSE", // -> else "NOP", - "JUMP", // -> end (skip else) - "NOP", // else body + "JUMP_NO_INTERRUPT", // -> end (skip else) + "NOP", // else body "LOAD_CONST", "RETURN_VALUE", } @@ -88,6 +88,7 @@ func TestForEmitsGetIterAndForIter(t *testing.T) { "LOAD_NAME", // xs "GET_ITER", "FOR_ITER", // -> cleanup + "NOP", // line-tracing NOP at the target (codegen_for) "STORE_NAME", // i (module scope) "NOP", // pass "JUMP", // -> for_iter diff --git a/compile/codegen_expr_call.go b/compile/codegen_expr_call.go index cedf3a47d..0f893a010 100644 --- a/compile/codegen_expr_call.go +++ b/compile/codegen_expr_call.go @@ -33,6 +33,16 @@ func (c *Compiler) visitCall(e *ast.Call) error { if ok { return nil } + // A call whose positional + keyword footprint exceeds the stack + // guideline routes through the CALL_FUNCTION_EX path so the + // arguments accumulate into a tuple / dict instead of piling onto + // the evaluation stack. + // + // CPython: Python/codegen.c:4258 codegen_call_helper_impl + // (nelts + nkwelts*2 > _PY_STACK_USE_GUIDELINE -> ex_call) + if len(e.Args)+len(e.Keywords)*2 > stackUseGuideline { + return c.emitCallEx(e) + } if hasKeyword(e.Keywords) { return c.emitCallKw(e) } @@ -81,14 +91,14 @@ func (c *Compiler) maybeOptimizeMethodCall(e *ast.Call) (bool, error) { pool := poolNames mangled := symtable.Mangle(c.unit().Private, attr.Attr) nameIdx := c.poolIndex(&pool, mangled) - c.addOpI(LOAD_ATTR, int32((nameIdx<<1)|1), loc(attr)) + c.addOpI(LOAD_ATTR, int32((nameIdx<<1)|1), updateStartLocationToMatchAttr(loc(attr), attr)) for _, a := range e.Args { if err := c.visitExpr(a); err != nil { return false, err } } if len(e.Keywords) == 0 { - c.addOpI(CALL, int32(len(e.Args)), loc(e)) + c.addOpI(CALL, int32(len(e.Args)), updateStartLocationToMatchAttr(loc(e), attr)) return true, nil } names := make([]any, 0, len(e.Keywords)) @@ -98,8 +108,8 @@ func (c *Compiler) maybeOptimizeMethodCall(e *ast.Call) (bool, error) { } names = append(names, *kw.Arg) } - c.addLoadConst(tupleOf(names), loc(e)) - c.addOpI(CALL_KW, int32(len(e.Args)+len(e.Keywords)), loc(e)) + c.addLoadConst(tupleOf(names), updateStartLocationToMatchAttr(loc(attr), attr)) + c.addOpI(CALL_KW, int32(len(e.Args)+len(e.Keywords)), updateStartLocationToMatchAttr(loc(e), attr)) return true, nil } @@ -176,32 +186,16 @@ func (c *Compiler) emitCallEx(e *ast.Call) error { return c.emitCallExKwargs(e) } } - c.addOpI(BUILD_LIST, 0, loc(e)) - pending := 0 - flushArgs := func() { - if pending == 0 { - return - } - c.addOpI(BUILD_LIST, int32(pending), loc(e)) - c.addOpI(LIST_EXTEND, 1, loc(e)) - pending = 0 - } - for _, a := range e.Args { - if star, ok := a.(*ast.Starred); ok { - flushArgs() - if err := c.visitExpr(star.Value); err != nil { - return err - } - c.addOpI(LIST_EXTEND, 1, loc(e)) - continue - } - if err := c.visitExpr(a); err != nil { - return err - } - pending++ + // Gather the positional args into a tuple, going through the shared + // starunpack helper so a large or starred list builds with a bounded + // evaluation stack (BUILD_LIST 0 + LIST_APPEND / LIST_EXTEND, then + // INTRINSIC_LIST_TO_TUPLE). + // + // CPython: Python/codegen.c codegen_call_helper_impl (ex_call -> + // starunpack_helper_impl with build=BUILD_LIST, tuple=1) + if err := c.emitStarunpack(e.Args, BUILD_LIST, LIST_APPEND, LIST_EXTEND, true, loc(e)); err != nil { + return err } - flushArgs() - c.addOpI(CALL_INTRINSIC_1, intrinsicListToTuple, loc(e)) return c.emitCallExKwargs(e) } @@ -215,37 +209,76 @@ func (c *Compiler) emitCallExKwargs(e *ast.Call) error { flag := int32(0) if hasKeyword(e.Keywords) { flag = 1 - c.addOpI(BUILD_MAP, 0, loc(e)) - pendingKw := 0 - flushKw := func() { - if pendingKw == 0 { - return - } - c.addOpI(BUILD_MAP, int32(pendingKw), loc(e)) - c.addOpI(DICT_MERGE, 1, loc(e)) - pendingKw = 0 - } - for _, kw := range e.Keywords { + kws := e.Keywords + haveDict := false + nseen := 0 + for i, kw := range kws { if kw.Arg == nil { - flushKw() + // A `**mapping` splat: pack up any pending run of plain + // keywords, then merge the mapping in. + if nseen != 0 { + if err := c.codegenSubkwargs(kws, i-nseen, i, loc(e)); err != nil { + return err + } + if haveDict { + c.addOpI(DICT_MERGE, 1, loc(e)) + } + haveDict = true + nseen = 0 + } + if !haveDict { + c.addOpI(BUILD_MAP, 0, loc(e)) + haveDict = true + } if err := c.visitExpr(kw.Value); err != nil { return err } c.addOpI(DICT_MERGE, 1, loc(e)) continue } - c.addLoadConst(*kw.Arg, loc(e)) - if err := c.visitExpr(kw.Value); err != nil { + nseen++ + } + if nseen != 0 { + if err := c.codegenSubkwargs(kws, len(kws)-nseen, len(kws), loc(e)); err != nil { return err } - pendingKw++ + if haveDict { + c.addOpI(DICT_MERGE, 1, loc(e)) + } + haveDict = true } - flushKw() } c.addOpI(CALL_FUNCTION_EX, flag, loc(e)) return nil } +// codegenSubkwargs emits the [begin,end) run of plain keyword arguments +// as one BUILD_MAP. A run whose stack footprint exceeds the guideline +// opens with BUILD_MAP 0 and folds each pair in with MAP_ADD so the +// value stack stays bounded. +// +// CPython: Python/codegen.c:4198 codegen_subkwargs +func (c *Compiler) codegenSubkwargs(kws ast.Seq[*ast.Keyword], begin, end int, l ast.Pos) error { + n := end - begin + big := n*2 > stackUseGuideline + if big { + c.addOpI(BUILD_MAP, 0, l) + } + for i := begin; i < end; i++ { + c.addLoadConst(*kws[i].Arg, l) + if err := c.visitExpr(kws[i].Value); err != nil { + return err + } + if big { + c.addOpI(MAP_ADD, 1, l) + } + } + if !big { + c.addOpI(BUILD_MAP, int32(n), l) + } + return nil +} + // validateKeywords rejects calls that pass the same keyword twice. // The diagnostic points at the second occurrence so the column matches // CPython's LOC(other) argument to _PyCompile_Error. diff --git a/compile/codegen_expr_comp.go b/compile/codegen_expr_comp.go index e1be036ad..54e3f857f 100644 --- a/compile/codegen_expr_comp.go +++ b/compile/codegen_expr_comp.go @@ -264,22 +264,41 @@ func (c *Compiler) compileGenerator(gens ast.Seq[*ast.Comprehension], ifCleanup := c.newLabel() anchor := c.newLabel() + // hasLoop tracks whether this generator emits a real FOR_ITER loop. + // The temporary-variable assignment idiom (`for y in [f(x)]`) elides + // the loop entirely: the single element is pushed and assigned once. + hasLoop := true + if !iterOnStack { if idx == 0 { // Implicit `.0` parameter holds the outermost iter. pool := poolVarNames c.addOpName(LOAD_FAST, &pool, ".0", loc(gen.Iter)) } else { - if err := c.visitExpr(gen.Iter); err != nil { - return err + // Fast path for the temporary variable assignment idiom: + // for y in [f(x)] + // A one-element list/tuple iter (non-starred) folds to a + // direct push + assign, dropping the FOR_ITER loop. + // CPython: Python/codegen.c:4420 + if elt := singleNonStarredElt(gen.Iter); elt != nil { + if err := c.visitExpr(elt); err != nil { + return err + } + hasLoop = false + } else { + if err := c.visitExpr(gen.Iter); err != nil { + return err + } + c.addOp(GET_ITER, loc(gen.Iter)) } - c.addOp(GET_ITER, loc(gen.Iter)) } } - depth++ - c.useLabel(start) - c.addOpJump(FOR_ITER, anchor, loc(gen.Iter)) + if hasLoop { + depth++ + c.useLabel(start) + c.addOpJump(FOR_ITER, anchor, loc(gen.Iter)) + } if err := c.assignTo(gen.Target, loc(gen.Target)); err != nil { return err } @@ -291,24 +310,57 @@ func (c *Compiler) compileGenerator(gens ast.Seq[*ast.Comprehension], c.addOpJump(POP_JUMP_IF_FALSE, ifCleanup, loc(ifx)) } + eltLoc := loc(elt) if idx+1 < len(gens) { if err := c.compileGenerator(gens, idx+1, depth, elt, val, kind, l, false); err != nil { return err } } else { - if err := c.emitCompTail(kind, depth, elt, val); err != nil { + tailLoc, err := c.emitCompTail(kind, depth, elt, val) + if err != nil { return err } + eltLoc = tailLoc } c.useLabel(ifCleanup) - c.addOpJump(JUMP, start, loc(gen.Iter)) - c.useLabel(anchor) - c.addOp(END_FOR, ast.Pos{}) - c.addOp(POP_ITER, ast.Pos{}) + if hasLoop { + // The loop-closing JUMP is located on the element, not the + // iterable, so the back-edge points at the produced value. + // + // CPython: Python/codegen.c:4508 ADDOP_JUMP(c, elt_loc, JUMP, start) + c.addOpJump(JUMP, start, eltLoc) + c.useLabel(anchor) + c.addOp(END_FOR, ast.Pos{}) + c.addOp(POP_ITER, ast.Pos{}) + } return nil } +// singleNonStarredElt returns the sole element of a one-element list or +// tuple display when that element is not a starred expression, else nil. +// It backs the comprehension assignment-idiom fast path. +// +// CPython: Python/codegen.c:4423 codegen_comprehension_generator +func singleNonStarredElt(iter ast.Expr) ast.Expr { + var elts ast.Seq[ast.Expr] + switch it := iter.(type) { + case *ast.List: + elts = it.Elts + case *ast.Tuple: + elts = it.Elts + default: + return nil + } + if len(elts) != 1 { + return nil + } + if _, ok := elts[0].(*ast.Starred); ok { + return nil + } + return elts[0] +} + // compileAsyncGenerator emits an async-for generator clause. // // CPython: Python/codegen.c:L4514 codegen_async_comprehension_generator @@ -323,9 +375,15 @@ func (c *Compiler) compileAsyncGenerator(gens ast.Seq[*ast.Comprehension], if !iterOnStack { if idx == 0 { + // gen_index 0 receives the outermost iter as the implicit + // `.0` argument; CPython locates LOAD_FAST at the whole- + // comprehension loc, not the iterable. + // + // CPython: Python/codegen.c:4546 ADDOP_I(c, loc, LOAD_FAST, 0) pool := poolVarNames - c.addOpName(LOAD_FAST, &pool, ".0", loc(gen.Iter)) + c.addOpName(LOAD_FAST, &pool, ".0", l) } else { + // CPython: Python/codegen.c:4544 ADDOP(c, LOC(gen->iter), GET_AITER) if err := c.visitExpr(gen.Iter); err != nil { return err } @@ -333,13 +391,17 @@ func (c *Compiler) compileAsyncGenerator(gens ast.Seq[*ast.Comprehension], } } + // The async-for scaffolding (SETUP_FINALLY .. END_ASYNC_FOR) carries + // the whole-comprehension loc in CPython, not the iterable's. + // + // CPython: Python/codegen.c:4549 codegen_async_comprehension_generator c.useLabel(start) c.pushFblock(fblockAsyncComprehensionGenerator, start, NoLabel, nil) - c.addOpJump(SETUP_FINALLY, except, loc(gen.Iter)) - c.addOp(GET_ANEXT, loc(gen.Iter)) - c.addLoadConst(nil, loc(gen.Iter)) - c.addYieldFromLoop(loc(gen.Iter)) - c.addOp(POP_BLOCK, loc(gen.Iter)) + c.addOpJump(SETUP_FINALLY, except, l) + c.addOp(GET_ANEXT, l) + c.addLoadConst(nil, l) + c.addYieldFromLoop(l) + c.addOp(POP_BLOCK, l) if err := c.assignTo(gen.Target, loc(gen.Target)); err != nil { return err @@ -353,63 +415,80 @@ func (c *Compiler) compileAsyncGenerator(gens ast.Seq[*ast.Comprehension], } depth++ + eltLoc := loc(elt) if idx+1 < len(gens) { if err := c.compileGenerator(gens, idx+1, depth, elt, val, kind, l, false); err != nil { return err } } else { - if err := c.emitCompTail(kind, depth, elt, val); err != nil { + tailLoc, err := c.emitCompTail(kind, depth, elt, val) + if err != nil { return err } + eltLoc = tailLoc } c.useLabel(ifCleanup) - c.addOpJump(JUMP, start, loc(gen.Iter)) + // CPython: Python/codegen.c:4611 ADDOP_JUMP(c, elt_loc, JUMP, start) + c.addOpJump(JUMP, start, eltLoc) if err := c.popFblock(fblockAsyncComprehensionGenerator); err != nil { return err } c.useLabel(except) - c.addOpJump(END_ASYNC_FOR, start, loc(gen.Iter)) + // CPython: Python/codegen.c:4617 ADDOP_JUMP(c, loc, END_ASYNC_FOR, send) + c.addOpJump(END_ASYNC_FOR, start, l) return nil } // emitCompTail writes the per-kind result accumulation step at the -// innermost generator depth. +// innermost generator depth. It returns the element location used for the +// accumulation opcode, which the caller reuses for the loop-closing JUMP. // // CPython: Python/codegen.c codegen_sync_comprehension_generator // (last-generator switch) -func (c *Compiler) emitCompTail(kind compKind, depth int, elt, val ast.Expr) error { +func (c *Compiler) emitCompTail(kind compKind, depth int, elt, val ast.Expr) (ast.Pos, error) { + eltLoc := loc(elt) switch kind { case compGenExp: if err := c.visitExpr(elt); err != nil { - return err + return eltLoc, err } // CPython: Python/codegen.c:4478 ADDOP_YIELD in comp_genexp tail. - c.addopYield(loc(elt)) - c.addOp(POP_TOP, loc(elt)) + c.addopYield(eltLoc) + c.addOp(POP_TOP, eltLoc) case compListComp: if err := c.visitExpr(elt); err != nil { - return err + return eltLoc, err } - c.addOpI(LIST_APPEND, int32(depth+1), loc(elt)) + c.addOpI(LIST_APPEND, int32(depth+1), eltLoc) case compSetComp: if err := c.visitExpr(elt); err != nil { - return err + return eltLoc, err } - c.addOpI(SET_ADD, int32(depth+1), loc(elt)) + c.addOpI(SET_ADD, int32(depth+1), eltLoc) case compDictComp: if err := c.visitExpr(elt); err != nil { - return err + return eltLoc, err } if err := c.visitExpr(val); err != nil { - return err + return eltLoc, err } - c.addOpI(MAP_ADD, int32(depth+1), loc(elt)) + // With '{k: v}', k is evaluated before v; the MAP_ADD location + // spans from the key's start to the value's end. + // + // CPython: Python/codegen.c:4496 elt_loc = LOCATION(...) + eltLoc = ast.Pos{ + Lineno: loc(elt).Lineno, + ColOffset: loc(elt).ColOffset, + EndLineno: loc(val).EndLineno, + EndColOffset: loc(val).EndColOffset, + } + c.addOpI(MAP_ADD, int32(depth+1), eltLoc) default: - return fmt.Errorf("compile: unknown comprehension kind %d", kind) + return eltLoc, fmt.Errorf("compile: unknown comprehension kind %d", kind) } - return nil + return eltLoc, nil } // wrapInStopIterationHandler bolts a SETUP_CLEANUP at offset 0 plus a @@ -421,11 +500,17 @@ func (c *Compiler) emitCompTail(kind compKind, depth int, elt, val ast.Expr) err // CPython: Python/codegen.c:L1175 codegen_wrap_in_stopiteration_handler func (c *Compiler) wrapInStopIterationHandler() { handler := c.newLabel() - c.seq().Insert(0, SETUP_CLEANUP, int32(handler.ID()), ast.Pos{}) + // All five ops carry NO_LOCATION (Lineno -1), not the zero Pos. A + // zero-lineno NOP would survive basicblock_remove_redundant_nops + // (which only drops NOPs with lineno < 0), leaving a stray entry NOP. + // + // CPython: Python/codegen.c:1175 codegen_wrap_in_stopiteration_handler + noLoc := ast.Pos{Lineno: -1} + c.seq().Insert(0, SETUP_CLEANUP, int32(handler.ID()), noLoc) - c.addLoadConst(nil, ast.Pos{}) - c.addOp(RETURN_VALUE, ast.Pos{}) + c.addLoadConst(nil, noLoc) + c.addOp(RETURN_VALUE, noLoc) c.useLabel(handler) - c.addOpI(CALL_INTRINSIC_1, intrinsicStopIterationError, ast.Pos{}) - c.addOpI(RERAISE, 1, ast.Pos{}) + c.addOpI(CALL_INTRINSIC_1, intrinsicStopIterationError, noLoc) + c.addOpI(RERAISE, 1, noLoc) } diff --git a/compile/codegen_expr_container.go b/compile/codegen_expr_container.go index c6b922f5a..eceb7287b 100644 --- a/compile/codegen_expr_container.go +++ b/compile/codegen_expr_container.go @@ -6,6 +6,7 @@ package compile import ( "fmt" + "unicode/utf8" "github.com/tamnd/gopy/ast" "github.com/tamnd/gopy/symtable" @@ -19,47 +20,40 @@ func (c *Compiler) visitList(e *ast.List) error { if e.Ctx != ast.Load { return fmt.Errorf("compile: List in non-load context handled by assignTo") } - return c.emitListOrSet(e.Elts, BUILD_LIST, loc(e)) + return c.emitStarunpack(e.Elts, BUILD_LIST, LIST_APPEND, LIST_EXTEND, false, loc(e)) } -// visitTuple emits a tuple literal in load context. +// visitTuple emits a tuple literal in load context. Like CPython's +// codegen_tuple it runs through starunpack_helper with tuple=1, so a +// large or starred tuple opens with BUILD_LIST and finishes with +// INTRINSIC_LIST_TO_TUPLE; only short, star-free tuples take the +// direct BUILD_TUPLE n form. // -// CPython: Python/codegen.c codegen_tuple +// CPython: Python/codegen.c:3449 codegen_tuple func (c *Compiler) visitTuple(e *ast.Tuple) error { if e.Ctx != ast.Load { return fmt.Errorf("compile: Tuple in non-load context handled by assignTo") } - if !hasStarred(e.Elts) { - for _, elt := range e.Elts { - if err := c.visitExpr(elt); err != nil { - return err - } - } - c.addOpI(BUILD_TUPLE, int32(len(e.Elts)), loc(e)) - return nil - } - // Tuple with stars: build a list, splat in, convert to tuple. - if err := c.emitListOrSet(e.Elts, BUILD_LIST, loc(e)); err != nil { - return err - } - c.addOpI(CALL_INTRINSIC_1, intrinsicListToTuple, loc(e)) - return nil + return c.emitStarunpack(e.Elts, BUILD_LIST, LIST_APPEND, LIST_EXTEND, true, loc(e)) } // visitSet emits a set literal. Stars splat with SET_UPDATE. // // CPython: Python/codegen.c:L3467 codegen_set func (c *Compiler) visitSet(e *ast.Set) error { - return c.emitListOrSet(e.Elts, BUILD_SET, loc(e)) + return c.emitStarunpack(e.Elts, BUILD_SET, SET_ADD, SET_UPDATE, false, loc(e)) } -// emitListOrSet handles the BUILD_LIST / BUILD_SET / BUILD_TUPLE-as-list -// branches by walking elts. The short, no-stars path emits the literal -// directly with `op n`; the big-or-starred path opens with `BUILD_LIST 0` -// and appends/extends per element so the value stack stays bounded. +// emitStarunpack walks elts and emits a list, set, or tuple display. +// The short, star-free path emits the literal directly: BUILD_TUPLE n +// when tuple is set, otherwise build n. The big-or-starred path opens +// with `build 0`, appends/extends per element to keep the value stack +// bounded, and (for tuples) finishes with INTRINSIC_LIST_TO_TUPLE. +// Tuples therefore pass build=BUILD_LIST so the spill path uses a list +// that the flowgraph can fold back into a constant tuple. // // CPython: Python/codegen.c:3318 starunpack_helper_impl -func (c *Compiler) emitListOrSet(elts ast.Seq[ast.Expr], op Opcode, l ast.Pos) error { +func (c *Compiler) emitStarunpack(elts ast.Seq[ast.Expr], build, add, extend Opcode, tuple bool, l ast.Pos) error { n := len(elts) seenStar := hasStarred(elts) big := n > stackUseGuideline @@ -69,44 +63,39 @@ func (c *Compiler) emitListOrSet(elts ast.Seq[ast.Expr], op Opcode, l ast.Pos) e return err } } - c.addOpI(op, int32(n), l) + if tuple { + c.addOpI(BUILD_TUPLE, int32(n), l) + } else { + c.addOpI(build, int32(n), l) + } return nil } - addOp := SET_ADD - extendOp := SET_UPDATE - if op != BUILD_SET { - addOp = LIST_APPEND - extendOp = LIST_EXTEND - } sequenceBuilt := false if big { - c.addOpI(op, 0, l) + c.addOpI(build, 0, l) sequenceBuilt = true } for i, elt := range elts { if star, ok := elt.(*ast.Starred); ok { if !sequenceBuilt { - c.addOpI(op, int32(i), l) + c.addOpI(build, int32(i), l) sequenceBuilt = true } if err := c.visitExpr(star.Value); err != nil { return err } - c.addOpI(extendOp, 1, l) + c.addOpI(extend, 1, l) continue } if err := c.visitExpr(elt); err != nil { return err } if sequenceBuilt { - c.addOpI(addOp, 1, l) + c.addOpI(add, 1, l) } } - if !sequenceBuilt { - // Pure non-star path already handled above; this would only - // trip if n == 0 with no stars, but the caller never invokes - // us for empty literals. Guard anyway. - c.addOpI(op, 0, l) + if tuple { + c.addOpI(CALL_INTRINSIC_1, intrinsicListToTuple, l) } return nil } @@ -139,68 +128,98 @@ const intrinsicListToTuple int32 = 6 // Include/internal/pycore_intrinsics.h). const intrinsicUnaryPositive int32 = 5 -// visitDict emits a dict literal. CPython lays this out as a series -// of LOAD_CONST keys + BUILD_MAP for each contiguous run, with -// DICT_UPDATE for `**other` splat keys. +// codegenSubdict emits the [begin,end) run of key/value pairs as one +// BUILD_MAP. A run whose stack footprint (2 entries per pair) exceeds +// the guideline opens with BUILD_MAP 0 and folds each pair in with +// MAP_ADD so the value stack stays bounded. // -// CPython: Python/codegen.c:L3497 codegen_dict +// CPython: Python/codegen.c:3474 codegen_subdict +func (c *Compiler) codegenSubdict(e *ast.Dict, begin, end int) error { + n := end - begin + big := n*2 > stackUseGuideline + if big { + c.addOpI(BUILD_MAP, 0, loc(e)) + } + for i := begin; i < end; i++ { + if err := c.visitExpr(e.Keys[i]); err != nil { + return err + } + if err := c.visitExpr(e.Values[i]); err != nil { + return err + } + if big { + c.addOpI(MAP_ADD, 1, loc(e)) + } + } + if !big { + c.addOpI(BUILD_MAP, int32(n), loc(e)) + } + return nil +} + +// visitDict emits a dict literal. Contiguous runs of non-splat keys are +// chunked through codegenSubdict so a large literal keeps a bounded +// evaluation stack; `**other` splat keys flush the pending run and fold +// in with DICT_UPDATE. +// +// CPython: Python/codegen.c:3496 codegen_dict func (c *Compiler) visitDict(e *ast.Dict) error { if len(e.Keys) != len(e.Values) { return fmt.Errorf("compile: Dict keys/values mismatch %d/%d", len(e.Keys), len(e.Values)) } - hasSplat := false - for _, k := range e.Keys { - if k == nil { - hasSplat = true - break - } - } - if !hasSplat { - for i, k := range e.Keys { - if err := c.visitExpr(k); err != nil { - return err + n := len(e.Values) + haveDict := false + elements := 0 + for i := range n { + isUnpacking := e.Keys[i] == nil + if isUnpacking { + if elements != 0 { + if err := c.codegenSubdict(e, i-elements, i); err != nil { + return err + } + if haveDict { + c.addOpI(DICT_UPDATE, 1, loc(e)) + } + haveDict = true + elements = 0 } - if err := c.visitExpr(e.Values[i]); err != nil { - return err - } - } - c.addOpI(BUILD_MAP, int32(len(e.Keys)), loc(e)) - return nil - } - // Splat path: BUILD_MAP 0, accumulate runs of non-splat keys, - // then DICT_UPDATE for each `**v`. - c.addOpI(BUILD_MAP, 0, loc(e)) - pending := 0 - flush := func() error { - if pending == 0 { - return nil - } - c.addOpI(BUILD_MAP, int32(pending), loc(e)) - c.addOpI(DICT_UPDATE, 1, loc(e)) - pending = 0 - return nil - } - for i, k := range e.Keys { - if k == nil { - if err := flush(); err != nil { - return err + if !haveDict { + c.addOpI(BUILD_MAP, 0, loc(e)) + haveDict = true } if err := c.visitExpr(e.Values[i]); err != nil { return err } c.addOpI(DICT_UPDATE, 1, loc(e)) - continue + } else { + if elements*2 > stackUseGuideline { + if err := c.codegenSubdict(e, i-elements, i+1); err != nil { + return err + } + if haveDict { + c.addOpI(DICT_UPDATE, 1, loc(e)) + } + haveDict = true + elements = 0 + } else { + elements++ + } } - if err := c.visitExpr(k); err != nil { + } + if elements != 0 { + if err := c.codegenSubdict(e, n-elements, n); err != nil { return err } - if err := c.visitExpr(e.Values[i]); err != nil { - return err + if haveDict { + c.addOpI(DICT_UPDATE, 1, loc(e)) } - pending++ + haveDict = true } - return flush() + if !haveDict { + c.addOpI(BUILD_MAP, 0, loc(e)) + } + return nil } // visitAttribute emits a LOAD_ATTR / STORE_ATTR / DELETE_ATTR @@ -221,22 +240,57 @@ func (c *Compiler) visitAttribute(e *ast.Attribute) error { // CPython: Python/codegen.c codegen_visit_expr (Attribute_kind, mangle branch) attr := symtable.Mangle(c.unit().Private, e.Attr) pool := poolNames + // The attribute opcodes are located on the attribute name, not the + // whole `value.attr` span, so a multi-line receiver does not drag the + // opcode's lineno back to where the receiver started. + // + // CPython: Python/codegen.c:5285 loc = update_start_location_to_match_attr(c, LOC(e), e) + l := updateStartLocationToMatchAttr(loc(e), e) switch e.Ctx { case ast.Load: // LOAD_ATTR oparg low bit is the "push self" hint used by // LOAD_METHOD; codegen leaves it clear and the flowgraph // optimizes it. - c.addOpName(LOAD_ATTR, &pool, attr, loc(e)) + c.addOpName(LOAD_ATTR, &pool, attr, l) case ast.Store: - c.addOpName(STORE_ATTR, &pool, attr, loc(e)) + c.addOpName(STORE_ATTR, &pool, attr, l) case ast.Del: - c.addOpName(DELETE_ATTR, &pool, attr, loc(e)) + c.addOpName(DELETE_ATTR, &pool, attr, l) default: return fmt.Errorf("compile: Attribute with unknown context %v", e.Ctx) } return nil } +// updateStartLocationToMatchAttr moves an attribute opcode's start +// location onto the attribute name's line. When the located span begins +// on a different line than the attribute ends (a receiver split across +// lines), the start is pulled to the attribute's end line and the start +// column is backed off by the attribute name's length so the opcode +// points at the name itself. Weird ASTs whose name is longer than the +// end column drop their columns, matching GH-94694. +// +// CPython: Python/codegen.c:3824 update_start_location_to_match_attr +func updateStartLocationToMatchAttr(l ast.Pos, e *ast.Attribute) ast.Pos { + if l.Lineno != e.Pos.EndLineno { + l.Lineno = e.Pos.EndLineno + n := utf8.RuneCountInString(e.Attr) + if n <= e.Pos.EndColOffset { + l.ColOffset = e.Pos.EndColOffset - n + } else { + l.ColOffset = -1 + l.EndColOffset = -1 + } + if l.Lineno > l.EndLineno { + l.EndLineno = l.Lineno + } + if l.Lineno == l.EndLineno && l.ColOffset > l.EndColOffset { + l.EndColOffset = l.ColOffset + } + } + return l +} + // maybeAddStaticAttribute walks the active unit stack and, when the // attribute is `self.X = ...` reached from inside a method, records X // on the nearest enclosing class unit. The class body emitter later @@ -260,11 +314,15 @@ func (c *Compiler) maybeAddStaticAttribute(e *ast.Attribute) { } } -// visitSubscript emits BINARY_SUBSCR / STORE_SUBSCR / DELETE_SUBSCR. -// Slice expressions go through visitSlice and still leave a Slice -// object on the stack. +// visitSubscript emits the load / store / delete sequence for a +// subscript. A non-constant two-part slice (`x[a:b]`, step omitted) +// takes the two-element fast path: the bounds are pushed bare and +// BINARY_SLICE / STORE_SLICE consume them without building a slice +// object. Every other subscript visits the slice expression (which +// for a constant slice is a LOAD_CONST of the slice object) and then +// folds into BINARY_OP NB_SUBSCR / STORE_SUBSCR / DELETE_SUBSCR. // -// CPython: Python/codegen.c:5548 codegen_subscript +// CPython: Python/codegen.c:5549 codegen_subscript func (c *Compiler) visitSubscript(e *ast.Subscript) error { if e.Ctx == ast.Load { if err := c.checkSubscripter(e.Value); err != nil { @@ -277,6 +335,20 @@ func (c *Compiler) visitSubscript(e *ast.Subscript) error { if err := c.visitExpr(e.Value); err != nil { return err } + if shouldApplyTwoElementSliceOptimization(e.Slice) && e.Ctx != ast.Del { + if err := c.codegenSliceTwoParts(e.Slice.(*ast.Slice)); err != nil { + return err + } + switch e.Ctx { + case ast.Load: + c.addOp(BINARY_SLICE, loc(e)) + case ast.Store: + c.addOp(STORE_SLICE, loc(e)) + default: + return fmt.Errorf("compile: Subscript with unknown context %v", e.Ctx) + } + return nil + } if err := c.visitExpr(e.Slice); err != nil { return err } @@ -294,15 +366,65 @@ func (c *Compiler) visitSubscript(e *ast.Subscript) error { return nil } -// visitSlice emits BUILD_SLICE 2 or 3 depending on whether step is -// present. Missing lower / upper become LOAD_CONST None. +// isConstantSlice reports whether every present bound of the slice is a +// constant literal. Absent bounds count as constant (they become None). // -// CPython: Python/codegen.c codegen_slice -func (c *Compiler) visitSlice(e *ast.Slice) error { - if err := c.visitOptExpr(e.Lower, loc(e)); err != nil { +// CPython: Python/codegen.c:5326 is_constant_slice +func isConstantSlice(s *ast.Slice) bool { + return isNilOrConstant(s.Lower) && + isNilOrConstant(s.Upper) && + isNilOrConstant(s.Step) +} + +// isNilOrConstant reports whether e is absent or a Constant node. +func isNilOrConstant(e ast.Expr) bool { + if e == nil { + return true + } + _, ok := e.(*ast.Constant) + return ok +} + +// shouldApplyTwoElementSliceOptimization reports whether a subscript's +// slice is a non-constant slice with no step, the shape BINARY_SLICE / +// STORE_SLICE handle directly. +// +// CPython: Python/codegen.c:5338 should_apply_two_element_slice_optimization +func shouldApplyTwoElementSliceOptimization(e ast.Expr) bool { + s, ok := e.(*ast.Slice) + if !ok { + return false + } + return !isConstantSlice(s) && s.Step == nil +} + +// codegenSliceTwoParts pushes the lower and upper bounds of a slice, +// each defaulting to LOAD_CONST None when absent. +// +// CPython: Python/codegen.c:5589 codegen_slice_two_parts +func (c *Compiler) codegenSliceTwoParts(s *ast.Slice) error { + if err := c.visitOptExpr(s.Lower, loc(s)); err != nil { return err } - if err := c.visitOptExpr(e.Upper, loc(e)); err != nil { + return c.visitOptExpr(s.Upper, loc(s)) +} + +// visitSlice emits a slice subscript index. A fully constant slice +// becomes a single LOAD_CONST of the slice object so the flowgraph can +// fold the enclosing NB_SUBSCR; otherwise the bounds (and optional +// step) are pushed and BUILD_SLICE 2 / 3 assembles them. +// +// CPython: Python/codegen.c:5610 codegen_slice +func (c *Compiler) visitSlice(e *ast.Slice) error { + if isConstantSlice(e) { + c.addLoadConst(&ConstSlice{ + Start: constBoundValue(e.Lower), + Stop: constBoundValue(e.Upper), + Step: constBoundValue(e.Step), + }, loc(e)) + return nil + } + if err := c.codegenSliceTwoParts(e); err != nil { return err } n := int32(2) @@ -316,6 +438,27 @@ func (c *Compiler) visitSlice(e *ast.Slice) error { return nil } +// constBoundValue returns the const value of a slice bound: nil (None) +// when the bound is absent, otherwise the Constant node's value. Only +// valid for bounds that satisfy isNilOrConstant. +// +// CPython: Python/codegen.c:5618 codegen_slice (Constant.value reads) +func constBoundValue(e ast.Expr) any { + if e == nil { + return nil + } + return e.(*ast.Constant).Value +} + +// ConstSlice is the codegen-side placeholder for a Python slice +// constant. The assembler / lift pass converts it to a real slice +// object during marshal; the VM lifts it at LOAD_CONST time. +// +// CPython: a constant slice is just a PySliceObject in co_consts. +type ConstSlice struct { + Start, Stop, Step any +} + // visitOptExpr visits an optional expression, emitting LOAD_CONST None // if it is missing. // diff --git a/compile/codegen_expr_misc.go b/compile/codegen_expr_misc.go index 20d03ece3..04a6cc0ff 100644 --- a/compile/codegen_expr_misc.go +++ b/compile/codegen_expr_misc.go @@ -43,9 +43,10 @@ func (c *Compiler) visitYield(e *ast.Yield) error { } // addopYield emits the wrap intrinsic when the scope is an async -// generator, then YIELD_VALUE. Centralizes the codegen so the -// comprehension elt-tail and the `yield` expression visit go through -// the same path, mirroring CPython's ADDOP_YIELD macro. +// generator, then YIELD_VALUE followed by the RESUME that marks the +// resume point after the generator is sent back into. Centralizes the +// codegen so the comprehension elt-tail and the `yield` expression +// visit go through the same path, mirroring CPython's ADDOP_YIELD macro. // // CPython: Python/codegen.c:3168 codegen_addop_yield func (c *Compiler) addopYield(l ast.Pos) { @@ -53,6 +54,7 @@ func (c *Compiler) addopYield(l ast.Pos) { c.addOpI(CALL_INTRINSIC_1, intrinsicAsyncGenWrap, l) } c.addOpI(YIELD_VALUE, 0, l) + c.addOpI(RESUME, resumeAfterYield, l) } // visitYieldFrom lowers `yield from x` to GET_YIELD_FROM_ITER plus a diff --git a/compile/codegen_expr_op.go b/compile/codegen_expr_op.go index cc51725fe..a7d72a731 100644 --- a/compile/codegen_expr_op.go +++ b/compile/codegen_expr_op.go @@ -11,15 +11,18 @@ import ( // visitBoolOp emits the short-circuit form for `and` / `or`. // -// A and B and C -> evaluate A; copy + to_bool; if false jump to -// end with the first falsy value on top. -// A or B or C -> same shape with POP_JUMP_IF_TRUE. +// A and B and C -> evaluate A; pseudo JUMP_IF_FALSE to end with the +// first falsy value on top; otherwise POP_TOP. +// A or B or C -> same shape with the pseudo JUMP_IF_TRUE. // -// CPython emits the pseudo JUMP_IF_FALSE / JUMP_IF_TRUE here and -// convert_pseudo_conditional_jumps later expands it to COPY 1 + TO_BOOL -// + POP_JUMP_IF_X. gopy emits the expanded form inline because its -// flowgraph has no pseudo-conditional pass; the resulting bytecode is -// identical. +// We emit the pseudo JUMP_IF_FALSE / JUMP_IF_TRUE exactly as CPython +// does. convert_pseudo_conditional_jumps later expands each into +// COPY 1 + TO_BOOL + POP_JUMP_IF_X. Keeping the pseudo form through the +// optimizer lets jump threading collapse `a and b or c`-style chains +// (gh-124285): a JUMP_IF_FALSE whose target re-tests the same value +// with the opposite-sense JUMP_IF_TRUE is threaded past the re-test. +// Emitting the expanded form here would hide those pseudo jumps from +// the threader and double-evaluate the operand. // // CPython: Python/codegen.c:3290 codegen_boolop // CPython: Python/flowgraph.c:3485 convert_pseudo_conditional_jumps @@ -28,9 +31,9 @@ func (c *Compiler) visitBoolOp(e *ast.BoolOp) error { return fmt.Errorf("compile: BoolOp needs at least two values") } end := c.newLabel() - jump := POP_JUMP_IF_FALSE + jump := JUMP_IF_FALSE if e.Op == ast.Or { - jump = POP_JUMP_IF_TRUE + jump = JUMP_IF_TRUE } for i, v := range e.Values { if err := c.visitExpr(v); err != nil { @@ -39,8 +42,6 @@ func (c *Compiler) visitBoolOp(e *ast.BoolOp) error { if i == len(e.Values)-1 { break } - c.addOpI(COPY, 1, loc(e)) - c.addOp(TO_BOOL, loc(e)) c.addOpJump(jump, end, loc(e)) c.addOp(POP_TOP, loc(e)) } diff --git a/compile/codegen_expr_test.go b/compile/codegen_expr_test.go index 050481d44..8b1958c80 100644 --- a/compile/codegen_expr_test.go +++ b/compile/codegen_expr_test.go @@ -21,14 +21,15 @@ func TestBoolOpAndShortCircuits(t *testing.T) { Values: []ast.Expr{nameLoad("x"), nameLoad("y")}, } u := compileMod(t, exprMod(e)) + // Codegen emits the pseudo JUMP_IF_FALSE; convert_pseudo_conditional_jumps + // in the CFG pass later expands it to COPY 1 + TO_BOOL + POP_JUMP_IF_FALSE. + // compileMod returns the pre-optimization unit, so the pseudo form shows. want := []string{ - "LOAD_NAME", // x - "COPY", // dup for jump check - "TO_BOOL", // normalize before POP_JUMP_IF_X - "POP_JUMP_IF_FALSE", // -> end (x falsy: leave x) - "POP_TOP", // discard the dup - "LOAD_NAME", // y - "POP_TOP", // ExprStmt discard + "LOAD_NAME", // x + "JUMP_IF_FALSE", // -> end (x falsy: leave x) + "POP_TOP", // discard x + "LOAD_NAME", // y + "POP_TOP", // ExprStmt discard "LOAD_CONST", "RETURN_VALUE", } if got := opNames(u); !equalStrings(got, want) { @@ -44,8 +45,10 @@ func TestBoolOpOrUsesPopJumpIfTrue(t *testing.T) { } u := compileMod(t, exprMod(e)) got := opNames(u) - if got[3] != "POP_JUMP_IF_TRUE" { - t.Errorf("expected POP_JUMP_IF_TRUE for `or`, got %s", got[3]) + // Raw codegen emits the pseudo JUMP_IF_TRUE (index 1, right after the + // LOAD_NAME for x); the CFG pass later expands it to POP_JUMP_IF_TRUE. + if got[1] != "JUMP_IF_TRUE" { + t.Errorf("expected JUMP_IF_TRUE for `or`, got %s", got[1]) } } @@ -326,17 +329,44 @@ func TestSubscriptLoadEmitsBinaryOpSubscr(t *testing.T) { } func TestSliceEmitsBuildSlice(t *testing.T) { - // x[1:2] + // x[a:b:c]. A three-part slice with a step still builds the slice on + // the stack via BUILD_SLICE; the two-element subscript optimization + // (BINARY_SLICE) only applies when there is no step. + // CPython: Python/codegen.c:5610 codegen_slice e := &ast.Subscript{ Value: nameLoad("x"), - Slice: &ast.Slice{Lower: cnst(int64(1)), Upper: cnst(int64(2))}, + Slice: &ast.Slice{Lower: nameLoad("a"), Upper: nameLoad("b"), Step: nameLoad("c")}, Ctx: ast.Load, } u := compileMod(t, exprMod(e)) got := opNames(u) wantPrefix := []string{ - "LOAD_NAME", "LOAD_CONST", "LOAD_CONST", "BUILD_SLICE", "BINARY_OP", + "LOAD_NAME", "LOAD_NAME", "LOAD_NAME", "LOAD_NAME", "BUILD_SLICE", "BINARY_OP", + } + if len(got) < len(wantPrefix) { + t.Fatalf("ops too short: %v", got) + } + for i, op := range wantPrefix { + if got[i] != op { + t.Errorf("ops[%d] = %s, want %s (full=%v)", i, got[i], op, got) + } } +} + +// TestConstantSliceFoldsToLoadConst pins the CPython 3.14 behavior where a +// fully constant slice subscript loads a slice constant and folds through +// NB_SUBSCR rather than emitting BUILD_SLICE. +// CPython: Python/codegen.c:5326 is_constant_slice +func TestConstantSliceFoldsToLoadConst(t *testing.T) { + // x[1:2] + e := &ast.Subscript{ + Value: nameLoad("x"), + Slice: &ast.Slice{Lower: cnst(int64(1)), Upper: cnst(int64(2))}, + Ctx: ast.Load, + } + u := compileMod(t, exprMod(e)) + got := opNames(u) + wantPrefix := []string{"LOAD_NAME", "LOAD_CONST", "BINARY_OP"} if len(got) < len(wantPrefix) { t.Fatalf("ops too short: %v", got) } diff --git a/compile/codegen_fblock.go b/compile/codegen_fblock.go index d0bdd3392..fee91b0ee 100644 --- a/compile/codegen_fblock.go +++ b/compile/codegen_fblock.go @@ -47,6 +47,12 @@ type fblock struct { // Datum carries the original AST node so // unwindFblock can re-evaluate context expressions. Datum any + // Loc mirrors CPython's fblockinfo.fb_loc. The with/async-with + // unwind restores it so the exit-protocol ops carry the location + // of the with statement rather than the unwinding statement. + // + // CPython: Python/codegen.c fblockinfo.fb_loc + Loc ast.Pos } // pushFblock pushes a frame block onto the active unit's stack. @@ -54,6 +60,12 @@ type fblock struct { // CPython: Python/codegen.c codegen_push_fblock func (c *Compiler) pushFblock(kind fblockKind, block, exit JumpTargetLabel, datum any) { c.fblocks = append(c.fblocks, fblock{Kind: kind, Block: block, Exit: exit, Datum: datum}) + // The FINALLY_END fblock guards the second (exception-path) copy of + // a finally body; suppress duplicate SyntaxWarnings while it is live. + // CPython: Python/compile.c:769 _PyCompile_PushFBlock + if kind == fblockFinallyEnd { + c.disableWarning++ + } } // popFblock pops the top frame block. Asserts the kind matches what @@ -70,6 +82,10 @@ func (c *Compiler) popFblock(kind fblockKind) error { c.fblocks[n-1].Kind, kind) } c.fblocks = c.fblocks[:n-1] + // CPython: Python/compile.c:783 _PyCompile_PopFBlock + if kind == fblockFinallyEnd { + c.disableWarning-- + } return nil } diff --git a/compile/codegen_stmt.go b/compile/codegen_stmt.go index 88c0b1e07..d2ca5a919 100644 --- a/compile/codegen_stmt.go +++ b/compile/codegen_stmt.go @@ -19,6 +19,16 @@ import ( // CPython: Python/codegen.c:L868 codegen_body called from // _PyCodegen_Module func (c *Compiler) visitModule(m *ast.Module) error { + // _PyCodegen_Module emits the conditional-annotations set BEFORE the + // body so __conditional_annotations__ is registered as names[0] and + // the set exists before the first annotated statement runs. The loc is + // start_location(stmts): the first statement's line, taken before the + // docstring is consumed. + // + // CPython: Python/codegen.c:858 _PyCodegen_Module + if err := c.emitConditionalAnnotationsPrologue(moduleStartLoc(m.Body)); err != nil { + return err + } docstring, docLoc, hasDoc := moduleDocstring(m.Body) body := c.consumeDocstring(m.Body) // CPython codegen_body emits LOAD_CONST / STORE_NAME @@ -35,26 +45,41 @@ func (c *Compiler) visitModule(m *ast.Module) error { c.addOpName(STORE_NAME, &pool, "__doc__", ast.Pos{Lineno: -1}) } // PEP 649: module annotations are deferred. visitAnnAssign records - // each annotation into the unit's DeferredAnnotations slice. - // After the body, annotation setup code (__conditional_annotations__ - // + __annotate__) is emitted into a separate stash sequence that - // cfgFromSequence prepends at the start of the body, so __annotate__ - // is available before any body statement executes. This matches - // CPython's _PyCompile_StartAnnotationSetup / - // _PyCompile_EndAnnotationSetup stash. + // each annotation into the unit's DeferredAnnotations slice. After the + // body the __annotate__ function is emitted into a separate stash + // sequence that cfgFromSequence prepends at the start of the body, so + // __annotate__ is available before any body statement executes. This + // matches CPython's _PyCompile_StartAnnotationSetup / + // _PyCompile_EndAnnotationSetup stash, which wraps only the + // __annotate__ build (the conditional set is emitted inline above). // // CPython: Python/compile.c _PyCompile_StartAnnotationSetup (L739) // CPython: Python/flowgraph.c:3946 _PyCfg_FromInstructionSequence if err := c.visitStmts(body); err != nil { return err } - if err := c.stashAnnotationCode(ast.Pos{Lineno: 1}); err != nil { + // codegen_process_deferred_annotations runs at the tail of codegen_body + // with the same start_location, so the stashed __annotate__ build keeps + // the first statement's line, not line 1. + if err := c.stashAnnotationCode(moduleStartLoc(m.Body)); err != nil { return err } c.addReturnNoneIfMissing(ast.Pos{Lineno: -1}) return nil } +// moduleStartLoc returns start_location(stmts): the location of the first +// statement, used for the conditional-annotations prologue. Falls back to +// line 1 for an empty body. +// +// CPython: Python/codegen.c:829 _PyCodegen_Module(c, start_location(stmts), ...) +func moduleStartLoc(body ast.Seq[ast.Stmt]) ast.Pos { + if len(body) == 0 { + return ast.Pos{Lineno: 1} + } + return loc(body[0]) +} + // visitInteractive is the REPL form. Every expression statement at // module nest level (including those nested inside `with`, `if`, // `for`, ... bodies) emits PRINT_EXPR so the interactive shell sees @@ -70,6 +95,9 @@ func (c *Compiler) visitInteractive(m *ast.Interactive) error { prev := c.interactive c.interactive = true defer func() { c.interactive = prev }() + if err := c.emitConditionalAnnotationsPrologue(moduleStartLoc(m.Body)); err != nil { + return err + } for _, s := range m.Body { if err := c.visitStmt(s); err != nil { return err @@ -83,7 +111,7 @@ func (c *Compiler) visitInteractive(m *ast.Interactive) error { // // CPython: Python/codegen.c:895 codegen_body (runs StartAnnotationSetup / // EndAnnotationSetup for the interactive branch as well) - if err := c.stashAnnotationCode(ast.Pos{Lineno: 1}); err != nil { + if err := c.stashAnnotationCode(moduleStartLoc(m.Body)); err != nil { return err } c.addReturnNoneIfMissing(ast.Pos{Lineno: -1}) @@ -305,10 +333,14 @@ func (c *Compiler) visitExprStmt(s *ast.ExprStmt) error { // CPython: Python/codegen.c codegen_stmt_expr (PRINT_EXPR // branch emits the intrinsic followed by POP_TOP) c.addOpI(CALL_INTRINSIC_1, intrinsicPrint, loc(s)) - c.addOp(POP_TOP, loc(s)) + c.addOp(POP_TOP, ast.Pos{Lineno: -1}) return nil } - c.addOp(POP_TOP, loc(s)) + // The POP_TOP that discards the expression result is artificial: + // emit it at NO_LOCATION so propagateLineNumbers inherits the value + // expression's line rather than the statement's opening line. + // CPython: Python/codegen.c:2978 codegen_stmt_expr (NO_LOCATION POP_TOP) + c.addOp(POP_TOP, ast.Pos{Lineno: -1}) return nil } @@ -345,7 +377,7 @@ func (c *Compiler) visitReturn(s *ast.Return) error { l = loc(s) c.addOp(NOP, l) } - c.unwindForReturn(preserveTOS, l) + c.unwindForReturn(preserveTOS, &l) if s.Value == nil { c.addLoadConst(nil, l) } else if !preserveTOS { diff --git a/compile/codegen_stmt_control.go b/compile/codegen_stmt_control.go index 86770f9a5..8d08cc3ac 100644 --- a/compile/codegen_stmt_control.go +++ b/compile/codegen_stmt_control.go @@ -27,7 +27,13 @@ func (c *Compiler) visitIf(s *ast.If) error { return err } if hasElse { - c.addOpJump(JUMP, end, loc(s)) + // The jump over the else arm is artificial: NO_LOCATION lets it + // inherit the line of the preceding body instruction via + // propagate_line_numbers, rather than stamping the if-statement + // line onto the duplicated scope-exit block. + // + // CPython: Python/codegen.c:2060 ADDOP_JUMP(c, NO_LOCATION, JUMP_NO_INTERRUPT, end) + c.addOpJump(JUMP_NO_INTERRUPT, end, ast.Pos{Lineno: -1}) c.useLabel(elseLbl) if err := c.visitStmts(s.Orelse); err != nil { return err @@ -61,7 +67,12 @@ func (c *Compiler) visitWhile(s *ast.While) error { if err := c.visitStmts(s.Body); err != nil { return err } - c.addOpJump(JUMP, loop, loc(s)) + // The loop backedge is artificial: NO_LOCATION lets it inherit the + // line of the body instruction it follows via propagate_line_numbers, + // instead of stamping the whole tail block with the while-test line. + // + // CPython: Python/codegen.c:2178 ADDOP_JUMP(c, NO_LOCATION, JUMP, loop) + c.addOpJump(JUMP, loop, ast.Pos{Lineno: -1}) c.useLabel(anchor) if err := c.popFblock(fblockWhileLoop); err != nil { @@ -103,6 +114,15 @@ func (c *Compiler) visitFor(s *ast.For) error { c.pushFblock(fblockForLoop, start, end, s) c.addOpJump(FOR_ITER, cleanup, iterLoc) + + // NOP carrying the target's location keeps line tracing correct for + // multiline for statements; remove_redundant_nops drops it later when + // it adds nothing. Without it a `pass` body's NOP and the artificial + // JUMP_BACKWARD mis-attribute their lines. + // + // CPython: Python/codegen.c:2091 ADDOP(c, LOC(s->v.For.target), NOP) + c.addOp(NOP, loc(s.Target)) + c.useLabel(body) if err := c.assignTo(s.Target, loc(s.Target)); err != nil { return err @@ -110,13 +130,19 @@ func (c *Compiler) visitFor(s *ast.For) error { if err := c.visitStmts(s.Body); err != nil { return err } - c.addOpJump(JUMP, start, loc(s)) + // The loop-back jump is artificial: NO_LOCATION lets it inherit the + // line of whatever precedes it (the removed body NOP donates forward). + // + // CPython: Python/codegen.c:2097 ADDOP_JUMP(c, NO_LOCATION, JUMP, start) + noLoc := ast.Pos{Lineno: -1} + c.addOpJump(JUMP, start, noLoc) - // CPython: Python/codegen.c:2101 END_FOR comes first so instrumentation - // can attach to it, then POP_ITER drops the exhausted iterator. + // CPython: Python/codegen.c:2105 END_FOR comes first so instrumentation + // can attach to it, then POP_ITER drops the exhausted iterator. Both + // carry NO_LOCATION and inherit their line via propagate_line_numbers. c.useLabel(cleanup) - c.addOp(END_FOR, loc(s)) - c.addOp(POP_ITER, loc(s)) + c.addOp(END_FOR, noLoc) + c.addOp(POP_ITER, noLoc) if err := c.popFblock(fblockForLoop); err != nil { return err } @@ -195,9 +221,9 @@ func (c *Compiler) visitBreak(s *ast.Break) error { if idx < 0 { return c.errorAt(l, "'break' outside loop") } - c.unwindToLoop(idx, l) + c.unwindToLoop(idx, &l) loop := &c.fblocks[idx] - c.unwindFblock(loop, l, false) + c.unwindFblock(loop, &l, false) c.addOpJump(JUMP, loop.Exit, l) return nil } @@ -211,7 +237,7 @@ func (c *Compiler) visitContinue(s *ast.Continue) error { if idx < 0 { return c.errorAt(l, "'continue' not properly in loop") } - c.unwindToLoop(idx, l) + c.unwindToLoop(idx, &l) c.addOpJump(JUMP, c.fblocks[idx].Block, l) return nil } @@ -221,9 +247,9 @@ func (c *Compiler) visitContinue(s *ast.Continue) error { // unwound. // // CPython: Python/codegen.c:L622 codegen_unwind_fblock_stack -func (c *Compiler) unwindToLoop(loopIdx int, l ast.Pos) { +func (c *Compiler) unwindToLoop(loopIdx int, ploc *ast.Pos) { for i := len(c.fblocks) - 1; i > loopIdx; i-- { - c.unwindFblock(&c.fblocks[i], l, false) + c.unwindFblock(&c.fblocks[i], ploc, false) } } @@ -234,9 +260,9 @@ func (c *Compiler) unwindToLoop(loopIdx int, l ast.Pos) { // popping (CPython's codegen_unwind_fblock preserve_tos branch). // // CPython: Python/codegen.c:L622 codegen_unwind_fblock_stack -func (c *Compiler) unwindForReturn(preserveTos bool, l ast.Pos) { +func (c *Compiler) unwindForReturn(preserveTos bool, ploc *ast.Pos) { for i := len(c.fblocks) - 1; i >= 0; i-- { - c.unwindFblock(&c.fblocks[i], l, preserveTos) + c.unwindFblock(&c.fblocks[i], ploc, preserveTos) } } @@ -244,7 +270,8 @@ func (c *Compiler) unwindForReturn(preserveTos bool, l ast.Pos) { // CPython's codegen_unwind_fblock. // // CPython: Python/codegen.c:L518 codegen_unwind_fblock -func (c *Compiler) unwindFblock(fb *fblock, l ast.Pos, preserveTos bool) { +func (c *Compiler) unwindFblock(fb *fblock, ploc *ast.Pos, preserveTos bool) { + l := *ploc switch fb.Kind { case fblockWhileLoop, fblockExceptionHandler, @@ -260,6 +287,7 @@ func (c *Compiler) unwindFblock(fb *fblock, l ast.Pos, preserveTos bool) { case fblockTryExcept: c.addOp(POP_BLOCK, l) case fblockFinallyTry: + // This POP_BLOCK gets the line number of the unwinding statement. c.addOp(POP_BLOCK, l) // Datum is ast.Seq[ast.Stmt], not []ast.Stmt. Go does not consider // named generic types (type Seq[T] []T) identical to their base @@ -303,6 +331,12 @@ func (c *Compiler) unwindFblock(fb *fblock, l ast.Pos, preserveTos bool) { } c.fblocks = origFblocks } + // The finally block should appear to execute after the statement + // causing the unwinding, so make the unwinding instruction + // artificial. + // + // CPython: Python/codegen.c:557 codegen_unwind_fblock FB_FINALLY_TRY + *ploc = noLocation case fblockFinallyEnd: if preserveTos { c.addOpI(SWAP, 2, l) @@ -314,6 +348,12 @@ func (c *Compiler) unwindFblock(fb *fblock, l ast.Pos, preserveTos bool) { c.addOp(POP_BLOCK, l) c.addOp(POP_EXCEPT, l) case fblockWith, fblockAsyncWith: + // The with exit ops carry the location of the with statement, + // not the unwinding statement. + // + // CPython: Python/codegen.c:570 codegen_unwind_fblock FB_WITH + *ploc = fb.Loc + l = *ploc c.addOp(POP_BLOCK, l) if preserveTos { // Each with-statement preamble pushes __exit__ AND its @@ -338,6 +378,11 @@ func (c *Compiler) unwindFblock(fb *fblock, l ast.Pos, preserveTos bool) { c.addYieldFromLoop(l) } c.addOp(POP_TOP, l) + // The exit block should appear to execute after the unwinding + // statement, so make the unwinding instruction artificial. + // + // CPython: Python/codegen.c:604 codegen_unwind_fblock FB_WITH + *ploc = noLocation case fblockHandlerCleanup: name, hasName := fb.Datum.(string) if hasName { diff --git a/compile/codegen_stmt_funclike.go b/compile/codegen_stmt_funclike.go index 9db87fd00..fd51446e0 100644 --- a/compile/codegen_stmt_funclike.go +++ b/compile/codegen_stmt_funclike.go @@ -39,7 +39,11 @@ func (c *Compiler) visitAsyncFunctionDef(s *ast.AsyncFunctionDef) error { // // CPython: Python/codegen.c:L1999 codegen_lambda func (c *Compiler) visitLambda(e *ast.Lambda) error { - body := ast.Seq[ast.Stmt]{&ast.Return{Value: e.Body, Pos: e.Pos}} + // The implicit RETURN_VALUE is located on the lambda body, not the + // whole `lambda ...:` span, so its position stays inside the body. + // + // CPython: Python/codegen.c:2027 loc = LOC(e->v.Lambda.body) + body := ast.Seq[ast.Stmt]{&ast.Return{Value: e.Body, Pos: loc(e.Body)}} return c.compileFunctionLike("", e.Args, body, nil, nil, nil, true, e) } @@ -59,6 +63,18 @@ func (c *Compiler) compileFunctionLike(name string, args *ast.Arguments, return err } + // A decorated function's co_firstlineno is the line of its first + // decorator, not the `def` line. enterScope derives firstlineno + // from the symtable entry (the def line), so override it for both + // the type-params wrapper and the function body below. + // + // CPython: Python/codegen.c:1420 codegen_function (firstlineno = + // decos[0]->lineno when the decorator list is non-empty) + funcFirstLineno := 0 + if len(decorators) > 0 { + funcFirstLineno = loc(decorators[0]).Lineno + } + // Evaluate defaults in the outer scope. They land on the stack // and are folded into MAKE_FUNCTION via the flags oparg. flags, err := c.emitDefaults(args, loc(scopeKey)) @@ -106,6 +122,9 @@ func (c *Compiler) compileFunctionLike(name string, args *ast.Arguments, outerCaches = c.savedCaches() c.enterScope(wrapperScope) + if funcFirstLineno > 0 { + c.unit().FirstLineno = funcFirstLineno + } first := c.unit().FirstLineno c.addOpI(RESUME, 0, ast.Pos{Lineno: first, EndLineno: first}) @@ -170,7 +189,7 @@ func (c *Compiler) compileFunctionLike(name string, args *ast.Arguments, } flags |= closureFlag - if err := c.emitInnerFunctionCode(innerScope, name, args, body, scopeKey); err != nil { + if err := c.emitInnerFunctionCode(innerScope, name, args, body, scopeKey, funcFirstLineno); err != nil { return err } @@ -243,6 +262,7 @@ func (c *Compiler) compileFunctionLike(name string, args *ast.Arguments, // CPython: Python/codegen.c:L1311 codegen_function_body func (c *Compiler) emitInnerFunctionCode(innerScope *symtable.Entry, name string, args *ast.Arguments, body ast.Seq[ast.Stmt], key any, + funcFirstLineno int, ) error { // Save the outer scope so we can restore it after the inner body. outerScope := c.scope @@ -251,6 +271,16 @@ func (c *Compiler) emitInnerFunctionCode(innerScope *symtable.Entry, c.enterScope(innerScope) + // A decorated function's co_firstlineno is the line of its first + // decorator, not the `def` line. enterScope derives firstlineno from + // the symtable entry (the def line), so override it here. + // + // CPython: Python/codegen.c:1420 codegen_function (firstlineno = + // decos[0]->lineno when the decorator list is non-empty) + if funcFirstLineno > 0 { + c.unit().FirstLineno = funcFirstLineno + } + // Pin the docstring at consts[0] before RESUME / declareArgs run, // so the first const slot belongs to the docstring (functions surface // __doc__ via co_consts[0] when CO_HAS_DOCSTRING is set). @@ -329,10 +359,13 @@ func (c *Compiler) emitInnerFunctionCode(innerScope *symtable.Entry, } // declareArgs registers each parameter in the per-unit varnames pool -// in declaration order: posonly, args, varargs, kwonly, kwargs. -// Mirrors CPython's compiler_arguments. +// in the same order the symtable adds them: posonly, args, kwonly, +// vararg, kwarg. This is what fixes co_varnames so *args / **kwargs +// land after the keyword-only slots, matching CPython's +// symtable_visit_arguments name order (which is what compile.c reads +// to build u_varnames). // -// CPython: Python/compile.c compiler_arguments +// CPython: Python/symtable.c:2884 symtable_visit_arguments func (c *Compiler) declareArgs(args *ast.Arguments) error { for _, a := range args.Posonlyargs { c.declareArg(a.Arg) @@ -340,12 +373,12 @@ func (c *Compiler) declareArgs(args *ast.Arguments) error { for _, a := range args.Args { c.declareArg(a.Arg) } - if args.Vararg != nil { - c.declareArg(args.Vararg.Arg) - } for _, a := range args.Kwonlyargs { c.declareArg(a.Arg) } + if args.Vararg != nil { + c.declareArg(args.Vararg.Arg) + } if args.Kwarg != nil { c.declareArg(args.Kwarg.Arg) } @@ -439,6 +472,11 @@ func (c *Compiler) emitMakeFunction(flags int32, l ast.Pos) { if flags&0x04 != 0 { c.addOpI(SET_FUNCTION_ATTRIBUTE, 0x04, l) } + // MAKE_FUNCTION_ANNOTATE (0x10): the PEP 649 __annotate__ callable. + // CPython: Python/codegen.c:950 codegen_make_closure + if flags&0x10 != 0 { + c.addOpI(SET_FUNCTION_ATTRIBUTE, 0x10, l) + } if flags&0x02 != 0 { c.addOpI(SET_FUNCTION_ATTRIBUTE, 0x02, l) } diff --git a/compile/codegen_stmt_misc.go b/compile/codegen_stmt_misc.go index 6665b4d57..b74b0a1a7 100644 --- a/compile/codegen_stmt_misc.go +++ b/compile/codegen_stmt_misc.go @@ -92,7 +92,11 @@ func (c *Compiler) visitAugAssign(s *ast.AugAssign) error { } c.addOpI(COPY, 1, targetLoc) pool := poolNames - c.addOpName(LOAD_ATTR, &pool, t.Attr, targetLoc) + // The attribute opcodes attach to the attribute name, not the + // receiver's first line. + // CPython: Python/codegen.c:5358 update_start_location_to_match_attr + attrLoc := updateStartLocationToMatchAttr(targetLoc, t) + c.addOpName(LOAD_ATTR, &pool, t.Attr, attrLoc) if err := c.visitExpr(s.Value); err != nil { return err } @@ -101,19 +105,30 @@ func (c *Compiler) visitAugAssign(s *ast.AugAssign) error { return err } c.addOpI(BINARY_OP, op, loc(s)) - c.addOpI(SWAP, 2, targetLoc) - c.addOpName(STORE_ATTR, &pool, t.Attr, targetLoc) + c.addOpI(SWAP, 2, attrLoc) + c.addOpName(STORE_ATTR, &pool, t.Attr, attrLoc) return nil case *ast.Subscript: if err := c.visitExpr(t.Value); err != nil { return err } - if err := c.visitExpr(t.Slice); err != nil { - return err + twoPart := shouldApplyTwoElementSliceOptimization(t.Slice) + if twoPart { + if err := c.codegenSliceTwoParts(t.Slice.(*ast.Slice)); err != nil { + return err + } + c.addOpI(COPY, 3, targetLoc) + c.addOpI(COPY, 3, targetLoc) + c.addOpI(COPY, 3, targetLoc) + c.addOp(BINARY_SLICE, targetLoc) + } else { + if err := c.visitExpr(t.Slice); err != nil { + return err + } + c.addOpI(COPY, 2, targetLoc) + c.addOpI(COPY, 2, targetLoc) + c.addOpI(BINARY_OP, nbSubscr, targetLoc) } - c.addOpI(COPY, 2, targetLoc) - c.addOpI(COPY, 2, targetLoc) - c.addOpI(BINARY_OP, nbSubscr, targetLoc) if err := c.visitExpr(s.Value); err != nil { return err } @@ -122,9 +137,16 @@ func (c *Compiler) visitAugAssign(s *ast.AugAssign) error { return err } c.addOpI(BINARY_OP, op, loc(s)) - c.addOpI(SWAP, 3, targetLoc) - c.addOpI(SWAP, 2, targetLoc) - c.addOp(STORE_SUBSCR, targetLoc) + if twoPart { + c.addOpI(SWAP, 4, targetLoc) + c.addOpI(SWAP, 3, targetLoc) + c.addOpI(SWAP, 2, targetLoc) + c.addOp(STORE_SLICE, targetLoc) + } else { + c.addOpI(SWAP, 3, targetLoc) + c.addOpI(SWAP, 2, targetLoc) + c.addOp(STORE_SUBSCR, targetLoc) + } return nil } return fmt.Errorf("compile: AugAssign target %T not supported", s.Target) diff --git a/compile/codegen_stmt_misc_test.go b/compile/codegen_stmt_misc_test.go index 7f0ad4568..b0b70242f 100644 --- a/compile/codegen_stmt_misc_test.go +++ b/compile/codegen_stmt_misc_test.go @@ -146,12 +146,16 @@ func TestAnnAssignWithValueAssigns(t *testing.T) { Simple: 1, } u := compileMod(t, module(a)) - // opNames prepends AnnoCode then body (minus RESUME). + // opNames prepends AnnoCode then body (minus RESUME). The __annotate__ + // stash is spliced at the placeholder (right after RESUME), so it lands + // before the body's BUILD_SET/STORE __conditional_annotations__ prologue, + // matching CPython's module emission order. want := []string{ - // AnnoCode: conditional-annotations prologue + __annotate__ setup - "BUILD_SET", "STORE_NAME", + // AnnoCode: __annotate__ setup (spliced at the placeholder) "LOAD_CONST", "MAKE_FUNCTION", "STORE_NAME", - // body: x = 1, then record condIdx in __conditional_annotations__ + // body: conditional-annotations prologue, then x = 1, then record + // condIdx in __conditional_annotations__ + "BUILD_SET", "STORE_NAME", "LOAD_CONST", "STORE_NAME", "LOAD_NAME", "LOAD_CONST", "SET_ADD", "POP_TOP", "LOAD_CONST", "RETURN_VALUE", @@ -174,10 +178,10 @@ func TestAnnAssignNoValueRecordsAnnotationOnly(t *testing.T) { } u := compileMod(t, module(a)) want := []string{ - // AnnoCode: conditional-annotations prologue + __annotate__ - "BUILD_SET", "STORE_NAME", + // AnnoCode: __annotate__ setup (spliced at the placeholder) "LOAD_CONST", "MAKE_FUNCTION", "STORE_NAME", - // body: record condIdx=0 in __conditional_annotations__ + // body: conditional-annotations prologue, then record condIdx=0 + "BUILD_SET", "STORE_NAME", "LOAD_NAME", "LOAD_CONST", "SET_ADD", "POP_TOP", "LOAD_CONST", "RETURN_VALUE", } diff --git a/compile/codegen_stmt_with.go b/compile/codegen_stmt_with.go index b24a61a44..ee171c452 100644 --- a/compile/codegen_stmt_with.go +++ b/compile/codegen_stmt_with.go @@ -46,6 +46,7 @@ func (c *Compiler) visitWithInner(s *ast.With, pos int) error { c.useLabel(block) c.pushFblock(fblockWith, block, final, s) + c.fblocks[len(c.fblocks)-1].Loc = l if item.OptionalVars != nil { if err := c.assignTo(item.OptionalVars, l); err != nil { @@ -126,6 +127,7 @@ func (c *Compiler) visitAsyncWithInner(s *ast.AsyncWith, pos int) error { c.useLabel(block) c.pushFblock(fblockAsyncWith, block, final, s) + c.fblocks[len(c.fblocks)-1].Loc = l if item.OptionalVars != nil { if err := c.assignTo(item.OptionalVars, l); err != nil { diff --git a/compile/codegen_test.go b/compile/codegen_test.go index 9d5c70854..26653eac7 100644 --- a/compile/codegen_test.go +++ b/compile/codegen_test.go @@ -54,6 +54,12 @@ func opNames(u *Unit) []string { if u.ScopeType == symtable.ModuleBlock && len(body) > 0 && body[0].Op == RESUME { body = body[1:] } + // Codegen plants ANNOTATIONS_PLACEHOLDER right after the module RESUME + // (cfgFromSequence later splices the __annotate__ stash in at it, or + // drops it). Skip it too so body-focused want lists keep matching. + if u.ScopeType == symtable.ModuleBlock && len(body) > 0 && body[0].Op == ANNOTATIONS_PLACEHOLDER { + body = body[1:] + } instrs = append(instrs, body...) out := make([]string, len(instrs)) for i, in := range instrs { diff --git a/compile/codegen_typeparams.go b/compile/codegen_typeparams.go index 3381da11f..648422604 100644 --- a/compile/codegen_typeparams.go +++ b/compile/codegen_typeparams.go @@ -116,6 +116,16 @@ func (c *Compiler) emitTypeParamDefault(expr ast.Expr, astNode any, slot int, l outerFblocks := c.fblocks outerCaches := c.savedCaches() + // CPython loads a (1,) defaults tuple in the outer scope before the + // closure const, then makes the closure with MAKE_FUNCTION_DEFAULTS so + // the lazy thunk's lone "format" parameter defaults to 1 (VALUE). + // + // CPython: Python/codegen.c:1199 codegen_type_param_bound_or_default + // (PyTuple_Pack(1, _PyLong_GetOne())). Emit the prebuilt (1,) const so + // it lands in co_consts ahead of the thunk code objects, matching + // CPython's const ordering. + c.addLoadConst(&ConstTuple{Values: []any{int64(1)}}, l) + closureFlag, err := c.emitClosure(innerScope, l) if err != nil { return err @@ -125,6 +135,22 @@ func (c *Compiler) emitTypeParamDefault(expr ast.Expr, astNode any, slot int, l first := c.unit().FirstLineno c.addOpI(RESUME, 0, ast.Pos{Lineno: first, EndLineno: first}) + // The thunk takes a single positional-only "format" argument and + // raises NotImplementedError for format > VALUE_WITH_FAKE_GLOBALS (2), + // matching codegen_setup_annotations_scope. annotationlib drives the + // FORWARDREF/STRING paths through the higher format values. + // + // CPython: Python/codegen.c:666 codegen_setup_annotations_scope + c.declareArg("format") + guardBody := c.newLabel() + c.addOpI(LOAD_FAST, 0, l) + c.addLoadConst(int64(2), l) // VALUE_WITH_FAKE_GLOBALS + c.addOpI(COMPARE_OP, int32(cmpGt), l) + c.addOpJump(POP_JUMP_IF_FALSE, guardBody, l) + c.addOpI(LOAD_COMMON_CONSTANT, constantNotImplementedError, l) + c.addOpI(RAISE_VARARGS, 1, l) + c.useLabel(guardBody) + // For starred TypeVarTuple defaults (*U), unwrap the starred value. if starExpr, ok := expr.(*ast.Starred); ok { if innerScope.Type == symtable.TypeVariableBlock { @@ -157,6 +183,11 @@ func (c *Compiler) emitTypeParamDefault(expr ast.Expr, astNode any, slot int, l c.addOp(RETURN_VALUE, l) innerUnit := c.unit() + // "format" is positional-only, like the __annotate__ scope. + // CPython: Python/codegen.c:669 codegen_setup_annotations_scope + innerUnit.Argcount = 0 + innerUnit.PosOnlyArgCount = 1 + innerUnit.KwOnlyArgCount = 0 c.leaveScope() c.scope = outerScope @@ -164,6 +195,8 @@ func (c *Compiler) emitTypeParamDefault(expr ast.Expr, astNode any, slot int, l c.restoreCaches(outerCaches) c.addLoadConst(innerUnit, l) - c.emitMakeFunction(closureFlag, l) + // MAKE_FUNCTION_DEFAULTS (0x01) attaches the (1,) defaults tuple so the + // thunk's "format" parameter defaults to 1 (VALUE). + c.emitMakeFunction(closureFlag|0x01, l) return nil } diff --git a/compile/compile_cfg_phase_hook.go b/compile/compile_cfg_phase_hook.go index 71a78adde..ee7f91263 100644 --- a/compile/compile_cfg_phase_hook.go +++ b/compile/compile_cfg_phase_hook.go @@ -93,7 +93,7 @@ func runCfgPhaseHook(unit *Unit, hook CompilePhaseHook) error { } consts := unit.Consts nlocals := len(unit.VarNames) - nparams := unit.PosOnlyArgCount + unit.Argcount + unit.KwOnlyArgCount + nparams := nparamsForUnit(unit) if err := cfgOptimizeCodeUnitWithHook(g, &consts, nlocals, nparams, unit.FirstLineno, phaseHook); err != nil { return fmt.Errorf("compile: %s: cfg pipeline: %w", unitName, err) } diff --git a/compile/compiler.go b/compile/compiler.go index 831d0b15a..ce6b66e85 100644 --- a/compile/compiler.go +++ b/compile/compiler.go @@ -14,6 +14,23 @@ import ( "github.com/tamnd/gopy/symtable" ) +// configOptimize is the interpreter's optimization level, the analog of +// PyConfig.optimization_level read by _Py_GetConfig(). The CLI sets it +// from -O / -OO / PYTHONOPTIMIZE before any source is compiled; a +// CompileFlags caller that passes optimize == -1 resolves the sentinel +// against this value, matching CPython's compiler_init. +// +// CPython: Python/compile.c:136 compiler_init (c_optimize fallback) +var configOptimize int + +// SetConfigOptimize records the interpreter optimization level. Called +// once during startup from the command-line / environment parse. +// +// CPython: Python/initconfig.c config_init_optimization_level +func SetConfigOptimize(level int) { + configOptimize = level +} + // Compile runs the full pipeline on a parsed module and returns the // top-level Code object plus any nested code objects (one per nested // scope). @@ -41,17 +58,20 @@ func Compile(mod ast.Mod, filename string, optimize int) (*Code, error) { // st_future->ff_features) func CompileFlags(mod ast.Mod, filename string, optimize, flags int) (*Code, error) { // optimize == -1 means "use the interpreter's optimization level". - // gopy has no -O switch yet, so that level is 0 (asserts kept, - // docstrings kept, __debug__ folds to True). Resolve it here so + // The CLI records -O / -OO (and PYTHONOPTIMIZE) into the package + // configOptimize level; resolve the sentinel against it here so // every downstream pass (preprocess __debug__ fold, assert removal, // docstring stripping) sees a non-negative level. // // CPython: Python/compile.c:353 _PyAST_Compile (optimize == -1 -> config->optimization_level) + if optimize < 0 { + optimize = configOptimize + } optimize = max(optimize, 0) // CPython: Python/compile.c:353 _PyAST_Compile — validate before any other pass. // CPython: Python/ast.c:1047 _PyAST_Validate if err := ast.Validate(mod); err != nil { - return nil, fmt.Errorf("ValueError: %s", err.Error()) + return nil, ast.WrapValidationError(err) } ff, err := future.FromAST(mod, filename) if err != nil { diff --git a/compile/compiler_assemble.go b/compile/compiler_assemble.go index ceeed43ea..b57fc521d 100644 --- a/compile/compiler_assemble.go +++ b/compile/compiler_assemble.go @@ -35,10 +35,28 @@ import "fmt" // // CPython: Python/compile.c:1411 optimize_and_assemble_code_unit // CPython: Python/compile.c:1429 nparams = PyList_GET_SIZE(u_ste->ste_varnames) +// nparamsForUnit counts the function's parameter slots, matching +// CPython's PyList_GET_SIZE(ste_varnames): the positional-only, regular, +// and keyword-only named parameters, plus the *args and **kwargs slots +// when present. The optimizer treats these slots as defined on entry, so +// loads of them borrow (LOAD_FAST_BORROW) rather than check. +// +// CPython: Python/compile.c:1429 nparams = PyList_GET_SIZE(u_ste->ste_varnames) +func nparamsForUnit(unit *Unit) int { + nparams := unit.PosOnlyArgCount + unit.Argcount + unit.KwOnlyArgCount + if unit.Flags&CoVarargs != 0 { + nparams++ + } + if unit.Flags&CoVarkeywords != 0 { + nparams++ + } + return nparams +} + func optimizeAndAssembleCodeUnit(unit *Unit, codeFlags uint32, filename string) (*Code, error) { g := cfgFromSequence(unit.Seq) nlocals := len(unit.VarNames) - nparams := unit.PosOnlyArgCount + unit.Argcount + unit.KwOnlyArgCount + nparams := nparamsForUnit(unit) if err := cfgOptimizeCodeUnit(g, &unit.Consts, nlocals, nparams, unit.FirstLineno); err != nil { return nil, fmt.Errorf("compile: %s: cfg optimize: %w", unit.Name, err) } diff --git a/compile/flowgraph_cfg.go b/compile/flowgraph_cfg.go index a209c1a8a..9912cff24 100644 --- a/compile/flowgraph_cfg.go +++ b/compile/flowgraph_cfg.go @@ -63,6 +63,28 @@ type basicblock struct { // // CPython: Python/flowgraph.c:189 basicblock_addop func (b *basicblock) addOp(op Opcode, oparg int32, loc ast.Pos) { + // CPython's basicblock_addop writes only i_opcode/i_oparg/i_target/i_loc + // and leaves i_except untouched (flowgraph.c:189). The instruction lands + // in the slot just past b_iused; basicblock arrays grow in place and the + // compaction passes (remove_redundant_nops) shrink b_iused with a memmove + // that does not clear the freed tail, so that slot frequently still holds + // a since-removed instruction of this same block, carrying that block's + // exception handler. Go slices behave the same way: remove_redundant_nops + // here reslices in place (flowgraph_cfg_passes.go), so the backing slot at + // the new length retains the former instruction's Except. Reusing the slot + // without a full struct literal preserves that stale handler, matching the + // protected fall-through CPython emits for a conditional jump inside a try + // or with body. When the slice has no spare capacity the append grows and + // the fresh tail is zeroed (Except == nil), mirroring C's realloc+memset. + if n := len(b.Instr); n < cap(b.Instr) { + b.Instr = b.Instr[:n+1] + slot := &b.Instr[n] + slot.Op = op + slot.Oparg = oparg + slot.Loc = loc + slot.Target = nil + return + } b.Instr = append(b.Instr, cfgInstr{Op: op, Oparg: oparg, Loc: loc}) } diff --git a/compile/flowgraph_cfg_bridge.go b/compile/flowgraph_cfg_bridge.go index 622e6f224..0845a6889 100644 --- a/compile/flowgraph_cfg_bridge.go +++ b/compile/flowgraph_cfg_bridge.go @@ -15,17 +15,18 @@ package compile // so subsequent flowgraph passes can ignore label ids entirely. // // When seq.AnnoCode is set (the PEP 649 annotation stash produced by -// stashAnnotationCode), its instructions are appended to g first so -// __annotate__ is defined before any body statement executes. +// stashAnnotationCode), its instructions replace the ANNOTATIONS_PLACEHOLDER +// pseudo that codegen planted right after the module RESUME, so __annotate__ +// is defined before any body statement executes but after RESUME. When no +// stash is present the placeholder is dropped instead. // // CPython: Python/flowgraph.c:3923 _PyCfg_FromInstructionSequence func cfgFromSequence(seq *Sequence) *cfgBuilder { seq.ApplyLabelMap(hasJumpTarget) - g := newCfgBuilder() if seq.AnnoCode != nil { seq.AnnoCode.ApplyLabelMap(hasJumpTarget) - appendSeqToGraph(g, seq.AnnoCode) } + g := newCfgBuilder() if len(seq.Instrs) > 0 { appendSeqToGraph(g, seq) } @@ -48,22 +49,45 @@ func appendSeqToGraph(g *cfgBuilder, seq *Sequence) { // to *basicblock pointers. // // After each jump, force the next instruction into a fresh block. - // CPython relies on IS_TERMINATOR_OPCODE (jumps OR scope exits) in - // cfg_builder_current_block_is_terminated, so every jump is always - // the last instruction in its block. gopy's narrower isTerminator - // predicate already handles scope exits; the explicit useNextBlock - // closes the gap for jumps so passes like cfgLabelExceptionTargets, - // which assume a jump terminates its block, see the same invariant. + // CPython terminates a block on IS_TERMINATOR_OPCODE, which is + // OPCODE_HAS_JUMP OR IS_SCOPE_EXIT_OPCODE. Crucially it does NOT + // include the block-push pseudos (SETUP_FINALLY / SETUP_WITH / + // SETUP_CLEANUP): those carry an exception-edge target but fall + // through to the protected region, which stays in the same block. + // gopy's narrower isTerminator predicate already handles scope + // exits; the explicit useNextBlock closes the gap for real jumps so + // passes like cfgLabelExceptionTargets, which assume a jump + // terminates its block, see the same invariant. We must mirror the + // OPCODE_HAS_JUMP boundary exactly: splitting after a SETUP_* would + // sever the with/try setup from its body, and optimize_load_fast + // would then see the bound __exit__ self as unconsumed at the false + // block end and refuse to emit LOAD_FAST_BORROW. idxToBlock := make([]*basicblock, len(seq.Instrs)) idxToInstr := make([]*cfgInstr, len(seq.Instrs)) for i, ins := range seq.Instrs { if isTarget[i] { g.useLabel(JumpTargetLabel{id: i + 1}) } + // ANNOTATIONS_PLACEHOLDER is the splice point for the stashed + // __annotate__ build. Expand the annotation code here (the stash + // is purely linear: no labels, no nested seqs, no jump targets), + // then drop the placeholder itself. With no stash it just vanishes. + // + // CPython: Python/flowgraph.c:3945 _PyCfg_FromInstructionSequence + if ins.Op == ANNOTATIONS_PLACEHOLDER { + if seq.AnnoCode != nil { + for _, ann := range seq.AnnoCode.Instrs { + g.addOp(ann.Op, ann.Oparg, ann.Loc) + } + } + idxToBlock[i] = g.CurBlock + idxToInstr[i] = g.CurBlock.lastInstr() + continue + } g.addOp(ins.Op, ins.Oparg, ins.Loc) idxToBlock[i] = g.CurBlock idxToInstr[i] = g.CurBlock.lastInstr() - if hasJumpTarget(ins.Op) { + if hasJumpTarget(ins.Op) && !isBlockPushOpcode(ins.Op) { g.useNextBlock(g.newBlock()) } } diff --git a/compile/flowgraph_cfg_constfold.go b/compile/flowgraph_cfg_constfold.go new file mode 100644 index 000000000..5762a530a --- /dev/null +++ b/compile/flowgraph_cfg_constfold.go @@ -0,0 +1,441 @@ +// Constant folding for the CFG optimizer: get_const_loading_instrs, +// nop_out, the const_folding_safe_* guards, and eval_const_binop / +// eval_const_unaryop. The evaluator drives the real runtime number and +// object protocols (objects.Number*, objects.GetItem) against the +// native const-pool values so float, complex, str, bytes, tuple, and +// subscript folds match CPython, not just the int64 cases. +// +// CPython: Python/flowgraph.c:1367 get_const_loading_instrs ff. + +package compile + +import ( + "math/big" + + "github.com/tamnd/gopy/ast" + "github.com/tamnd/gopy/objects" //nolint:depguard // const folding evaluates the real number/object protocols +) + +// const_folding size guards. +// +// CPython: Python/flowgraph.c:1690 MAX_INT_SIZE ff. +const ( + maxIntSize = 128 // bits + maxCollectionSize = 256 // items + maxStrSize = 4096 // characters + maxTotalItems = 1024 // including nested collections +) + +// cfgGetConstLoadingInstrs walks backward from start collecting size +// const-loading instructions, skipping NOPs. Returns the indices in +// operand order (instrs[0] is the first operand pushed) and ok=false if +// any non-NOP, non-const instruction is hit before size are found. +// +// CPython: Python/flowgraph.c:1367 get_const_loading_instrs +func cfgGetConstLoadingInstrs(bb *basicblock, start, size int) ([]int, bool) { + if size < 0 { + return nil, false + } + idxs := make([]int, size) + remaining := size + for s := start; s >= 0 && remaining > 0; s-- { + ins := &bb.Instr[s] + if ins.Op == NOP { + continue + } + if _, ok := cfgLoadsConstValue(ins, nil); !ok && ins.Op != LOAD_CONST && ins.Op != LOAD_SMALL_INT { + return nil, false + } + remaining-- + idxs[remaining] = s + } + if remaining != 0 { + return nil, false + } + return idxs, true +} + +// cfgNopOut turns each listed instruction into a NOP at NO_LOCATION. +// +// CPython: Python/flowgraph.c:1391 nop_out +func cfgNopOut(bb *basicblock, idxs []int) { + for _, k := range idxs { + bb.Instr[k].Op = NOP + bb.Instr[k].Oparg = 0 + bb.Instr[k].Target = nil + bb.Instr[k].Loc = noLocation + } +} + +// cfgConstToObject lifts a native const-pool value into the Python +// object the folding protocols operate on. ok=false for shapes that are +// never const-foldable operands (nested code objects, unknown types). +func cfgConstToObject(v any) (objects.Object, bool) { + switch x := v.(type) { + case nil: + return objects.None(), true + case bool: + return objects.NewBool(x), true + case int64: + return objects.NewInt(x), true + case int: + return objects.NewInt(int64(x)), true + case int32: + return objects.NewInt(int64(x)), true + case *big.Int: + return objects.NewIntFromBig(x), true + case float64: + return objects.NewFloat(x), true + case complex128: + return objects.NewComplex(real(x), imag(x)), true + case string: + return objects.NewStr(x), true + case []byte: + return objects.NewBytes(x), true + case ast.EllipsisType: + return objects.Ellipsis(), true + case *ConstTuple: + items := make([]objects.Object, len(x.Values)) + for i, e := range x.Values { + o, ok := cfgConstToObject(e) + if !ok { + return nil, false + } + items[i] = o + } + return objects.NewTuple(items), true + case *ConstSlice: + start, ok := cfgConstToObject(x.Start) + if !ok { + return nil, false + } + stop, ok := cfgConstToObject(x.Stop) + if !ok { + return nil, false + } + step, ok := cfgConstToObject(x.Step) + if !ok { + return nil, false + } + return objects.NewSlice(start, stop, step), true + case ast.FrozenSet: + items := make([]objects.Object, len(x)) + for i, e := range x { + o, ok := cfgConstToObject(e) + if !ok { + return nil, false + } + items[i] = o + } + fs, err := objects.NewFrozenset(items) + if err != nil { + return nil, false + } + return fs, true + case objects.Object: + return x, true + } + return nil, false +} + +// cfgObjectToConst is the inverse of cfgConstToObject: it turns a folded +// result object back into the native const-pool value. ok=false for +// objects that cannot live in the const pool. +func cfgObjectToConst(o objects.Object) (any, bool) { + switch x := o.(type) { + case *objects.Bool: + v, _ := x.Int64() + return v != 0, true + case *objects.Int: + if v, ok := x.Int64(); ok { + return v, true + } + return x.BigInt(), true + case *objects.Float: + return x.Float64(), true + case *objects.Complex: + return x.Complex128(), true + case *objects.Unicode: + return x.Value(), true + case *objects.Bytes: + return x.Bytes(), true + case *objects.Tuple: + vals := make([]any, x.Len()) + for i := 0; i < x.Len(); i++ { + c, ok := cfgObjectToConst(x.Item(i)) + if !ok { + return nil, false + } + vals[i] = c + } + return &ConstTuple{Values: vals}, true + } + if objects.IsNone(o) { + return nil, true + } + if o == objects.Ellipsis() { + return ast.EllipsisType{}, true + } + if x, ok := o.(*objects.Set); ok && o.Type() == objects.FrozensetType { + items := x.Items() + vals := make(ast.FrozenSet, len(items)) + for i, it := range items { + c, ok := cfgObjectToConst(it) + if !ok { + return nil, false + } + vals[i] = c + } + return vals, true + } + return nil, false +} + +// objAsInt returns the *objects.Int when o is an int (but not a bool, to +// mirror PyLong_Check excluding the seq*int guards never seeing bools as +// counts in practice; bool is a subtype so PyLong_Check is true, kept). +func objAsInt(o objects.Object) (*objects.Int, bool) { + switch x := o.(type) { + case *objects.Bool: + return &x.Int, true + case *objects.Int: + return x, true + } + return nil, false +} + +// objNumBits returns the bit length of |v| for an int object. +// +// CPython: Objects/longobject.c _PyLong_NumBits +func objNumBits(i *objects.Int) int64 { + if v, ok := i.Int64(); ok { + if v < 0 { + v = -v + } + return int64(bigLen64(uint64(v))) + } + b := new(big.Int).Abs(i.BigInt()) + return int64(b.BitLen()) +} + +func bigLen64(v uint64) int { + n := 0 + for v > 0 { + v >>= 1 + n++ + } + return n +} + +func objSeqLen(o objects.Object) (int, bool) { + switch x := o.(type) { + case *objects.Tuple: + return x.Len(), true + case *objects.Unicode: + return x.Length(), true + case *objects.Bytes: + return x.Len(), true + } + return 0, false +} + +// constFoldingCheckComplexity returns a negative number when the total +// item count of the (possibly nested) tuple exceeds limit. +// +// CPython: Python/flowgraph.c:1674 const_folding_check_complexity +func constFoldingCheckComplexity(o objects.Object, limit int) int { + if t, ok := o.(*objects.Tuple); ok { + limit -= t.Len() + for i := 0; limit >= 0 && i < t.Len(); i++ { + limit = constFoldingCheckComplexity(t.Item(i), limit) + if limit < 0 { + return limit + } + } + } + return limit +} + +// constFoldingSafeMultiply reports whether v*w is small enough to fold. +// +// CPython: Python/flowgraph.c:1696 const_folding_safe_multiply +func constFoldingSafeMultiply(v, w objects.Object) bool { + vi, vIsInt := objAsInt(v) + wi, wIsInt := objAsInt(w) + switch { + case vIsInt && wIsInt && vi.Sign() != 0 && wi.Sign() != 0: + if objNumBits(vi)+objNumBits(wi) > maxIntSize { + return false + } + case vIsInt: + if size, ok := objSeqLen(w); ok && size > 0 { + n, ok := vi.Int64() + if !ok || n < 0 || n > int64(maxCollectionSize/size) { + return false + } + if _, isTuple := w.(*objects.Tuple); isTuple && n != 0 && + constFoldingCheckComplexity(w, maxTotalItems/int(n)) < 0 { + return false + } + if _, isTuple := w.(*objects.Tuple); !isTuple && n > int64(maxStrSize/size) { + return false + } + } + case wIsInt: + if _, ok := objSeqLen(v); ok { + return constFoldingSafeMultiply(w, v) + } + } + return true +} + +// constFoldingSafePower reports whether v**w is small enough to fold. +// +// CPython: Python/flowgraph.c:1740 const_folding_safe_power +func constFoldingSafePower(v, w objects.Object) bool { + vi, vIsInt := objAsInt(v) + wi, wIsInt := objAsInt(w) + if vIsInt && wIsInt && vi.Sign() != 0 && wi.Sign() > 0 { + wbits, ok := wi.Int64() + if !ok || wbits <= 0 { + return false + } + if uint64(objNumBits(vi)) > uint64(maxIntSize)/uint64(wbits) { + return false + } + } + return true +} + +// constFoldingSafeLshift reports whether v< maxIntSize || uint64(objNumBits(vi)) > uint64(maxIntSize)-uint64(wbits) { + return false + } + } + return true +} + +func objIsStrOrBytes(o objects.Object) bool { + switch o.(type) { + case *objects.Unicode, *objects.Bytes: + return true + } + return false +} + +// cfgEvalConstBinop folds a BINARY_OP over two const operands, returning +// ok=false when the operands are not foldable, the result would be too +// large, or the operation raises (matching CPython's PyErr_Clear path). +// +// CPython: Python/flowgraph.c:1790 eval_const_binop +func cfgEvalConstBinop(leftC any, op int32, rightC any) (any, bool) { + left, ok := cfgConstToObject(leftC) + if !ok { + return nil, false + } + right, ok := cfgConstToObject(rightC) + if !ok { + return nil, false + } + var ( + res objects.Object + err error + ) + switch op { + case NB_ADD, NB_INPLACE_ADD: + res, err = objects.NumberAdd(left, right) + case NB_SUBTRACT, NB_INPLACE_SUBTRACT: + res, err = objects.NumberSubtract(left, right) + case NB_MULTIPLY, NB_INPLACE_MULTIPLY: + if !constFoldingSafeMultiply(left, right) { + return nil, false + } + res, err = objects.NumberMultiply(left, right) + case NB_TRUE_DIVIDE, NB_INPLACE_TRUE_DIVIDE: + res, err = objects.NumberTrueDivide(left, right) + case NB_FLOOR_DIVIDE, NB_INPLACE_FLOOR_DIVIDE: + res, err = objects.NumberFloorDivide(left, right) + case NB_REMAINDER, NB_INPLACE_REMAINDER: + if objIsStrOrBytes(left) { + return nil, false + } + res, err = objects.NumberRemainder(left, right) + case NB_POWER, NB_INPLACE_POWER: + if !constFoldingSafePower(left, right) { + return nil, false + } + res, err = objects.NumberPower(left, right, objects.None()) + case NB_LSHIFT, NB_INPLACE_LSHIFT: + if !constFoldingSafeLshift(left, right) { + return nil, false + } + res, err = objects.NumberLshift(left, right) + case NB_RSHIFT, NB_INPLACE_RSHIFT: + res, err = objects.NumberRshift(left, right) + case NB_OR, NB_INPLACE_OR: + res, err = objects.NumberOr(left, right) + case NB_XOR, NB_INPLACE_XOR: + res, err = objects.NumberXor(left, right) + case NB_AND, NB_INPLACE_AND: + res, err = objects.NumberAnd(left, right) + case NB_SUBSCR: + res, err = objects.GetItem(left, right) + default: + return nil, false + } + if err != nil || res == nil { + return nil, false + } + return cfgObjectToConst(res) +} + +// cfgEvalConstUnaryop folds a unary opcode over a const operand. +// +// CPython: Python/flowgraph.c:1893 eval_const_unaryop +func cfgEvalConstUnaryop(operandC any, op Opcode, oparg int32) (any, bool) { + operand, ok := cfgConstToObject(operandC) + if !ok { + return nil, false + } + var ( + res objects.Object + err error + ) + switch op { + case UNARY_NEGATIVE: + res, err = objects.NumberNegative(operand) + case UNARY_INVERT: + // ~bool stays unfolded until the deprecation expires. + if _, isBool := operand.(*objects.Bool); isBool { + return nil, false + } + res, err = objects.NumberInvert(operand) + case UNARY_NOT: + t, terr := objects.IsTruthy(operand) + if terr != nil { + return nil, false + } + return !t, true + case CALL_INTRINSIC_1: + if oparg != intrinsicUnaryPositive { + return nil, false + } + res, err = objects.NumberPositive(operand) + default: + return nil, false + } + if err != nil || res == nil { + return nil, false + } + return cfgObjectToConst(res) +} diff --git a/compile/flowgraph_cfg_passes.go b/compile/flowgraph_cfg_passes.go index bcac6d714..6763259ce 100644 --- a/compile/flowgraph_cfg_passes.go +++ b/compile/flowgraph_cfg_passes.go @@ -5,36 +5,11 @@ package compile import ( - "math/bits" + "fmt" "github.com/tamnd/gopy/ast" ) -// maxIntFoldBits caps the bit-length of folded integer results. CPython -// runs unlimited-precision arithmetic and refuses to fold once the -// product / power / shift exceeds 128 bits; gopy stores constants as -// int64, so the effective ceiling is 63 (sign bit reserved). -// -// CPython: Python/flowgraph.c:1690 MAX_INT_SIZE -const maxIntFoldBits = 63 - -// In-place BINARY_OP suboperators not declared in codegen_expr_op.go. -// Values come from CPython's NB_INPLACE_* enum. -// -// CPython: Include/opcode.h NB_INPLACE_ADD etc. -const ( - nbInplAdd int32 = 13 - nbInplAnd int32 = 14 - nbInplLsh int32 = 16 - nbInplMul int32 = 18 - nbInplOr int32 = 22 - nbInplRsh int32 = 24 - nbInplSub int32 = 23 - nbInplXor int32 = 25 -) - -const minInt64 int64 = -1 << 63 - // minConstSequenceSize mirrors CPython's MIN_CONST_SEQUENCE_SIZE: a // list / set literal shorter than this stays as N pushes + BUILD_X // because the LIST_EXTEND prelude would not pay off. @@ -49,170 +24,6 @@ const minConstSequenceSize = 3 // CPython: Python/flowgraph.c:50 NO_LOCATION var noLocation = ast.Pos{Lineno: -1, EndLineno: -1, ColOffset: -1, EndColOffset: -1} -// evalIntBinop computes the result of x y for integer operands, -// or returns ok=false if the operator is one we do not fold (TRUE_DIVIDE, -// MATRIX_MULTIPLY, FLOOR_DIVIDE on a zero divisor, POWER with a -// negative exponent, etc.). -// -// CPython: Python/flowgraph.c:1791 eval_const_binop -func evalIntBinop(op int32, x, y int64) (int64, bool) { - switch op { - case nbAdd, nbInplAdd: - return safeAdd(x, y) - case nbSubtract, nbInplSub: - return safeSub(x, y) - case nbMult, nbInplMul: - return safeMultiply(x, y) - case nbAnd, nbInplAnd: - return x & y, true - case nbOr, nbInplOr: - return x | y, true - case nbXor, nbInplXor: - return x ^ y, true - case nbLShift, nbInplLsh: - return safeLshift(x, y) - case nbRShift, nbInplRsh: - if y < 0 || y >= 64 { - return 0, false - } - return x >> uint(y), true - case nbPower: - return safePower(x, y) - case nbFloorDiv: - if y == 0 { - return 0, false - } - q := x / y - if (x%y != 0) && ((x < 0) != (y < 0)) { - q-- - } - return q, true - case nbRemainder: - if y == 0 { - return 0, false - } - r := x % y - if r != 0 && ((r < 0) != (y < 0)) { - r += y - } - return r, true - } - return 0, false -} - -// intBitLen reports the bit length of |v|, matching _PyLong_NumBits. -// -// CPython: Objects/longobject.c _PyLong_NumBits -func intBitLen(v int64) int { - if v < 0 { - v = -v - } - return bits.Len64(uint64(v)) -} - -// safeAdd returns x + y when the result fits in int64. Matches -// CPython's implicit "fits in MAX_INT_SIZE bits" guard for the -// gopy int64 const pool. -// -// CPython: Python/flowgraph.c:1799 PyNumber_Add (eval_const_binop) -func safeAdd(x, y int64) (int64, bool) { - r := x + y - if (y > 0 && r < x) || (y < 0 && r > x) { - return 0, false - } - return r, true -} - -// safeSub returns x - y when the result fits in int64. -// -// CPython: Python/flowgraph.c:1802 PyNumber_Subtract (eval_const_binop) -func safeSub(x, y int64) (int64, bool) { - r := x - y - if (y > 0 && r > x) || (y < 0 && r < x) { - return 0, false - } - return r, true -} - -// safeMultiply mirrors const_folding_safe_multiply for the int64 -// path. CPython caps at 128 bits combined; we cap at 63 to keep the -// result in int64. -// -// CPython: Python/flowgraph.c:1696 const_folding_safe_multiply -func safeMultiply(x, y int64) (int64, bool) { - if x == 0 || y == 0 { - return 0, true - } - if intBitLen(x)+intBitLen(y) > maxIntFoldBits { - return 0, false - } - return x * y, true -} - -// safeLshift mirrors const_folding_safe_lshift for the int64 path. -// -// CPython: Python/flowgraph.c:1761 const_folding_safe_lshift -func safeLshift(x, y int64) (int64, bool) { - if y < 0 { - return 0, false - } - if x == 0 || y == 0 { - return x, true - } - if y > maxIntFoldBits || intBitLen(x)+int(y) > maxIntFoldBits { - return 0, false - } - return x << uint(y), true -} - -// safePower mirrors const_folding_safe_power for the int64 path. Only -// folds non-negative exponents; negative exponents would produce a -// float, and gopy keeps the const pool homogeneous per slot. -// -// CPython: Python/flowgraph.c:1741 const_folding_safe_power -func safePower(x, y int64) (int64, bool) { - if y < 0 { - return 0, false - } - if x == 0 { - if y == 0 { - return 1, true - } - return 0, true - } - if y == 0 { - return 1, true - } - xbits := intBitLen(x) - if xbits == 0 { - xbits = 1 - } - if int64(xbits)*y > int64(maxIntFoldBits) { - return 0, false - } - result := int64(1) - base := x - exp := y - for exp > 0 { - if exp&1 == 1 { - r, ok := safeMultiply(result, base) - if !ok { - return 0, false - } - result = r - } - exp >>= 1 - if exp > 0 { - r, ok := safeMultiply(base, base) - if !ok { - return 0, false - } - base = r - } - } - return result, true -} - // appendConst returns the index of v in *consts, appending if not // present. Dedup runs through constCacheKey, the port of CPython's // _PyCode_ConstantKey, so nested *ConstTuple values share a slot when @@ -248,45 +59,6 @@ func isFoldableUnary(op Opcode, oparg int32) bool { return false } -// evalConstUnaryop applies one unary opcode to a const operand, mirroring -// CPython's PyNumber_Negative / PyNumber_Invert / bool(!x) / -// PyNumber_Positive dispatch. Returns ok=false when the operand type -// is not foldable (e.g. unary negate of a string) or would overflow the -// int64 representation. The CALL_INTRINSIC_1 case is only valid for -// oparg == INTRINSIC_UNARY_POSITIVE; callers gate via isFoldableUnary. -// -// CPython: Python/flowgraph.c:1894 eval_const_unaryop -func evalConstUnaryop(op Opcode, operand any) (any, bool) { - switch op { - case UNARY_NEGATIVE: - switch v := operand.(type) { - case int64: - if v == minInt64 { - return nil, false - } - return -v, true - case float64: - return -v, true - } - case UNARY_INVERT: - if v, ok := operand.(int64); ok { - return ^v, true - } - case UNARY_NOT: - b, ok := constTruthValue(operand) - if !ok { - return nil, false - } - return !b, true - case CALL_INTRINSIC_1: - switch operand.(type) { - case int64, float64, complex128: - return operand, true - } - } - return nil, false -} - // constTruthValue mirrors PyObject_IsTrue for the const-pool value // shapes gopy stores. Returns ok=false when the value's truthiness is // not statically determinable. @@ -399,8 +171,16 @@ func normalizeJumpsInBlock(g *cfgBuilder, b *basicblock) { } if !last.Target.Visited { - // Forward conditional: mark the fall-through edge with NOT_TAKEN - // so a later assembler pass can record it precisely. + // Forward conditional: append a NOT_TAKEN marker on the + // fall-through edge. addOp mirrors basicblock_addop and leaves + // the reused slot's Except in place, so a NOT_TAKEN landing on a + // slot vacated by NOP compaction inherits that block's handler + // (protected fall-through), while one landing on a freshly grown + // slot is born unprotected. This reproduces CPython's behaviour, + // where the new instruction's i_except is whatever the basicblock + // array slot happened to hold. + // + // CPython: Python/flowgraph.c:546 basicblock_addop(b, NOT_TAKEN, ...) b.addOp(NOT_TAKEN, 0, last.Loc) return } @@ -539,7 +319,15 @@ func cfgPropagateLineNumbers(g *cfgBuilder) { } } last := b.lastInstr() - if hasJumpTarget(last.Op) && last.Target != nil { + // CPython propagate_line_numbers keys on is_jump(last), NOT the + // is_jump||is_block_push predicate used for predecessor counting. + // A block whose last op is SETUP_FINALLY/SETUP_CLEANUP/SETUP_WITH + // reaches its handler only via the exception edge, so its handler + // block (PUSH_EXC_INFO and the cleanup arms) must keep + // NO_LOCATION rather than inheriting the setup's line. + // + // CPython: Python/flowgraph.c:3640 propagate_line_numbers + if isJumpOpcode(last.Op) && last.Target != nil { target := last.Target if target.Predecessors == 1 && len(target.Instr) > 0 && target.Instr[0].Loc.Lineno < 0 { target.Instr[0].Loc = prev @@ -560,7 +348,11 @@ func cfgDuplicateExitsWithoutLineno(g *cfgBuilder) { nextLbl := getMaxLabel(g) + 1 for b := g.EntryBlock; b != nil; b = b.Next { last := b.lastInstr() - if last == nil || !hasJumpTarget(last.Op) || last.Target == nil { + // is_jump(last), not is_jump||is_block_push: SETUP_* exception + // edges do not trigger exit-block duplication. + // + // CPython: Python/flowgraph.c:3574 duplicate_exits_without_lineno + if last == nil || !isJumpOpcode(last.Op) || last.Target == nil { continue } target := nextNonemptyBlock(last.Target) @@ -587,13 +379,44 @@ func cfgDuplicateExitsWithoutLineno(g *cfgBuilder) { } } -// isExitWithoutLineno mirrors is_exit_or_eval_check_without_lineno. -// gopy has no opcodes carrying the HAS_EVAL_BREAK_FLAG yet, so the -// eval-break leg of the disjunction is always false. +// opcodeHasEvalBreak mirrors OPCODE_HAS_EVAL_BREAK for the opcodes that +// can appear at flowgraph-optimization time (pseudo and base ops, before +// specialization). These are the ops the eval loop polls the eval +// breaker on: the unconditional jumps, the loop backedge, the calls, and +// RESUME. A block containing one of these is an "eval check" whose line +// number PEP 626 requires to be valid after the frame terminates. +// +// CPython: Include/internal/pycore_opcode_metadata.h:1049 OPCODE_HAS_EVAL_BREAK +func opcodeHasEvalBreak(op Opcode) bool { + switch op { + case JUMP, JUMP_BACKWARD, CALL, CALL_FUNCTION_EX, RESUME: + return true + } + return false +} + +// basicblockHasEvalBreak reports whether any instruction in b polls the +// eval breaker. +// +// CPython: Python/flowgraph.c:347 basicblock_has_eval_break +func basicblockHasEvalBreak(b *basicblock) bool { + for i := range b.Instr { + if opcodeHasEvalBreak(b.Instr[i].Op) { + return true + } + } + return false +} + +// isExitWithoutLineno mirrors is_exit_or_eval_check_without_lineno: a +// block that either exits the scope (return/raise/reraise) or contains an +// eval-breaker check (jump backedge, call, resume), and carries no line +// number on any instruction. Such blocks are duplicated so each jump into +// them owns a private copy that can inherit its predecessor's line. // // CPython: Python/flowgraph.c:3543 is_exit_or_eval_check_without_lineno func isExitWithoutLineno(b *basicblock) bool { - if !b.exitsScope() { + if !b.exitsScope() && !basicblockHasEvalBreak(b) { return false } for i := range b.Instr { @@ -1397,17 +1220,19 @@ func basicblockFoldConstUnaryop(bb *basicblock, consts *[]any) int { if !isFoldableUnary(ins.Op, ins.Oparg) { continue } - operand, ok := cfgLoadsConstValue(&bb.Instr[i-1], *consts) + operands, ok := cfgGetConstLoadingInstrs(bb, i-1, 1) if !ok { continue } - result, ok := evalConstUnaryop(ins.Op, operand) + operand, ok := cfgLoadsConstValue(&bb.Instr[operands[0]], *consts) if !ok { continue } - bb.Instr[i-1].Op = NOP - bb.Instr[i-1].Oparg = 0 - bb.Instr[i-1].Target = nil + result, ok := cfgEvalConstUnaryop(operand, ins.Op, ins.Oparg) + if !ok { + continue + } + cfgNopOut(bb, operands) cfgInstrMakeLoadConst(ins, result, consts) folded++ } @@ -1433,6 +1258,29 @@ func cfgLoadsConstValue(ins *cfgInstr, consts []any) (any, bool) { return nil, false } +// cfgGetConstValue resolves the constant loaded by opcode/oparg. A +// LOAD_CONST oparg outside the consts pool raises ValueError, matching +// CPython's get_const_value which the load-const and fold passes call +// for every const-loading instruction. LOAD_SMALL_INT returns its oparg +// as an int. +// +// CPython: Python/flowgraph.c:1294 get_const_value +func cfgGetConstValue(opcode Opcode, oparg int32, consts []any) (any, error) { + switch opcode { + case LOAD_CONST: + n := len(consts) + if oparg < 0 || int(oparg) >= n { + return nil, cfgError(fmt.Sprintf( + "ValueError: LOAD_CONST index %d is out of range for consts (len=%d)", + oparg, n)) + } + return consts[oparg], nil + case LOAD_SMALL_INT: + return int64(oparg), nil + } + return nil, cfgError("SystemError: Internal error: failed to get value of a constant") +} + // optimizeBasicBlockCFG runs CPython's per-block peephole/const-fold // pass over bb. Each switch case mirrors a case in CPython's // optimize_basic_block in source order; the trailing loop reruns @@ -1719,61 +1567,34 @@ func basicblockFoldConstBinop(bb *basicblock, consts *[]any) int { return 0 } folded := 0 - for i := 0; i+2 < len(bb.Instr); i++ { - a := &bb.Instr[i] - b := &bb.Instr[i+1] - c := &bb.Instr[i+2] + for i := 0; i < len(bb.Instr); i++ { + c := &bb.Instr[i] if c.Op != BINARY_OP { continue } - va, okA := cfgLoadsConstValue(a, *consts) - if !okA { + operands, ok := cfgGetConstLoadingInstrs(bb, i-1, 2) + if !ok { continue } - vb, okB := cfgLoadsConstValue(b, *consts) - if !okB { + va, okA := cfgLoadsConstValue(&bb.Instr[operands[0]], *consts) + if !okA { continue } - x, xok := va.(int64) - y, yok := vb.(int64) - if !xok || !yok { + vb, okB := cfgLoadsConstValue(&bb.Instr[operands[1]], *consts) + if !okB { continue } - result, ok := evalIntBinop(c.Oparg, x, y) + result, ok := cfgEvalConstBinop(va, c.Oparg, vb) if !ok { continue } - a.Op = NOP - a.Oparg = 0 - a.Target = nil - b.Op = NOP - b.Oparg = 0 - b.Target = nil + cfgNopOut(bb, operands) cfgInstrMakeLoadConst(c, result, consts) folded++ - i += 2 } return folded } -// basicblockCollectConstLoaders walks n instructions starting at start -// and returns their const values when every slot is a LOAD_CONST or -// LOAD_SMALL_INT. Within a basic block no slot can be a jump target, -// so the flat-substrate pinned gate is unnecessary. -// -// CPython: Python/flowgraph.c:1430 get_const_loading_instrs -func basicblockCollectConstLoaders(bb *basicblock, consts []any, start, n int) (bool, []any) { - values := make([]any, 0, n) - for k := range n { - v, ok := cfgLoadsConstValue(&bb.Instr[start+k], consts) - if !ok { - return false, nil - } - values = append(values, v) - } - return true, values -} - // basicblockFoldTupleOfConstants rewrites `LOAD_CONST c1; ...; // LOAD_CONST cN; BUILD_TUPLE N` into `NOP; ...; NOP; LOAD_CONST // (c1, ..., cN)`. @@ -1790,21 +1611,28 @@ func basicblockFoldTupleOfConstants(bb *basicblock, consts *[]any) int { continue } n := int(ins.Oparg) - if n <= 0 || i < n { + if n > stackUseGuideline { continue } - start := i - n - ok, values := basicblockCollectConstLoaders(bb, *consts, start, n) + operands, ok := cfgGetConstLoadingInstrs(bb, i-1, n) if !ok { continue } - tuple := &ConstTuple{Values: append([]any(nil), values...)} - for k := start; k < i; k++ { - bb.Instr[k].Op = NOP - bb.Instr[k].Oparg = 0 - bb.Instr[k].Target = nil - bb.Instr[k].Loc = noLocation + values := make([]any, n) + bad := false + for k, idx := range operands { + v, vok := cfgLoadsConstValue(&bb.Instr[idx], *consts) + if !vok { + bad = true + break + } + values[k] = v + } + if bad { + continue } + tuple := &ConstTuple{Values: values} + cfgNopOut(bb, operands) idx := appendConst(consts, tuple) ins.Op = LOAD_CONST ins.Oparg = int32(idx) @@ -1819,7 +1647,7 @@ func basicblockFoldTupleOfConstants(bb *basicblock, consts *[]any) int { // // CPython: Python/flowgraph.c:1597 optimize_lists_and_sets func basicblockOptimizeListsAndSets(bb *basicblock, consts *[]any) int { - if consts == nil || len(bb.Instr) < 3 { + if consts == nil || len(bb.Instr) == 0 { return 0 } folded := 0 @@ -1828,29 +1656,63 @@ func basicblockOptimizeListsAndSets(bb *basicblock, consts *[]any) int { if ins.Op != BUILD_LIST && ins.Op != BUILD_SET { continue } + var nextop Opcode + if i+1 < len(bb.Instr) { + nextop = bb.Instr[i+1].Op + } + // A literal list/set feeding a "for" loop, comprehension or + // "in"/"not in" test can become an immutable tuple/frozenset + // constant regardless of length; otherwise the prelude rewrite + // only pays off at MIN_CONST_SEQUENCE_SIZE elements. + containsOrIter := nextop == GET_ITER || nextop == CONTAINS_OP n := int(ins.Oparg) - if n < minConstSequenceSize || i < n { + if n > stackUseGuideline || (n < minConstSequenceSize && !containsOrIter) { continue } - start := i - n - ok, values := basicblockCollectConstLoaders(bb, *consts, start, n) + operands, ok := cfgGetConstLoadingInstrs(bb, i-1, n) if !ok { + // Not a const sequence: a list feeding for/in can still drop + // to a tuple, which is cheaper to build than a list. + if containsOrIter && ins.Op == BUILD_LIST { + ins.Op = BUILD_TUPLE + } continue } - tuple := &ConstTuple{Values: append([]any(nil), values...)} - idx := appendConst(consts, tuple) - for k := start; k < i-2; k++ { - bb.Instr[k].Op = NOP - bb.Instr[k].Oparg = 0 - bb.Instr[k].Target = nil - bb.Instr[k].Loc = noLocation + values := make([]any, n) + bad := false + for k, idx := range operands { + v, vok := cfgLoadsConstValue(&bb.Instr[idx], *consts) + if !vok { + bad = true + break + } + values[k] = v + } + if bad { + continue + } + var constResult any + if ins.Op == BUILD_SET { + constResult = ast.FrozenSet(values) + } else { + constResult = &ConstTuple{Values: values} + } + idx := appendConst(consts, constResult) + cfgNopOut(bb, operands) + if containsOrIter { + ins.Op = LOAD_CONST + ins.Oparg = int32(idx) + folded++ + continue } preludeOp := ins.Op bb.Instr[i-2].Op = preludeOp bb.Instr[i-2].Oparg = 0 + bb.Instr[i-2].Target = nil bb.Instr[i-2].Loc = ins.Loc bb.Instr[i-1].Op = LOAD_CONST bb.Instr[i-1].Oparg = int32(idx) + bb.Instr[i-1].Target = nil bb.Instr[i-1].Loc = ins.Loc if preludeOp == BUILD_LIST { ins.Op = LIST_EXTEND @@ -2150,12 +2012,15 @@ func cfgInstrMakeLoadConst(inst *cfgInstr, newconst any, consts *[]any) { // CPython: Python/flowgraph.c:2168 basicblock_optimize_load_const // //nolint:gocognit,gocyclo // direct port of the CPython switch; flattening hurts the 1:1 mapping with flowgraph.c:2168. -func basicblockOptimizeLoadConst(bb *basicblock, consts *[]any) { +func basicblockOptimizeLoadConst(bb *basicblock, consts *[]any) error { var opcode Opcode var oparg int32 for i := 0; i < len(bb.Instr); i++ { inst := &bb.Instr[i] if inst.Op == LOAD_CONST { + if _, err := cfgGetConstValue(inst.Op, inst.Oparg, *consts); err != nil { + return err + } maybeInstrMakeLoadSmallint(inst, *consts) } isCopyOfLoadConst := opcode == LOAD_CONST && inst.Op == COPY && inst.Oparg == 1 @@ -2241,16 +2106,20 @@ func basicblockOptimizeLoadConst(bb *basicblock, consts *[]any) { bb.Instr[i+1].Oparg = int32(idx) } } + return nil } // cfgOptimizeLoadConst is the outer driver: runs // basicblockOptimizeLoadConst over every block. // // CPython: Python/flowgraph.c:2301 optimize_load_const -func cfgOptimizeLoadConst(g *cfgBuilder, consts *[]any) { +func cfgOptimizeLoadConst(g *cfgBuilder, consts *[]any) error { for b := g.EntryBlock; b != nil; b = b.Next { - basicblockOptimizeLoadConst(b, consts) + if err := basicblockOptimizeLoadConst(b, consts); err != nil { + return err + } } + return nil } // setNopCfg clears a cfgInstr to a plain NOP, preserving Loc so the @@ -2346,7 +2215,9 @@ func cfgOptimizeCfg(g *cfgBuilder, consts *[]any, firstlineno int) error { cfgInlineSmallOrNoLinenoBlocks(g) cfgRemoveUnreachable(g) cfgResolveLineNumbers(g, firstlineno) - cfgOptimizeLoadConst(g, consts) + if err := cfgOptimizeLoadConst(g, consts); err != nil { + return err + } for b := g.EntryBlock; b != nil; b = b.Next { optimizeBasicBlockCFG(b, consts) } diff --git a/compile/flowgraph_cfg_stackdepth.go b/compile/flowgraph_cfg_stackdepth.go index 22fa3aea7..edc5753aa 100644 --- a/compile/flowgraph_cfg_stackdepth.go +++ b/compile/flowgraph_cfg_stackdepth.go @@ -150,6 +150,39 @@ func getStackEffects(op Opcode, oparg int32, jump bool, effects *stackEffects) e return nil } +// InvalidStackEffect marks an opcode/oparg pair with no defined stack +// effect, mirroring PY_INVALID_STACK_EFFECT (INT_MAX). +// +// CPython: Include/cpython/compile.h:48 PY_INVALID_STACK_EFFECT +const InvalidStackEffect = int(^uint32(0) >> 1) + +// OpcodeStackEffectWithJump returns the net stack effect of opcode for +// the given oparg. jump selects the branch column: -1 (None, the +// default), 0 (not taken), or 1 (taken). Unknown opcodes or opargs +// return InvalidStackEffect. +// +// CPython: Python/flowgraph.c:4086 PyCompile_OpcodeStackEffectWithJump +func OpcodeStackEffectWithJump(opcode int, oparg int32, jump int) int { + if opcode < 0 || opcode > maxPseudoOpcode { + return InvalidStackEffect + } + op := Opcode(opcode) + // Specialized instructions (FOR_ITER_LIST, LOAD_ATTR_SLOT, ...) carry + // no arity row and fall out of getStackEffects as invalid, matching + // CPython's _PyOpcode_Deopt[opcode] != opcode early-out. + var effects stackEffects + if err := getStackEffects(op, oparg, jump != 0, &effects); err != nil { + return InvalidStackEffect + } + return effects.Net +} + +// maxPseudoOpcode bounds the pseudo-opcode range; IS_VALID_OPCODE +// accepts opcode < 267. +// +// CPython: Include/internal/pycore_opcode_metadata.h IS_VALID_OPCODE +const maxPseudoOpcode = 266 + // isBlockPushOpcode mirrors IS_BLOCK_PUSH_OPCODE: SETUP_FINALLY, // SETUP_WITH, SETUP_CLEANUP push a try/with handler onto the block // stack and contribute a non-zero effect only when the implicit @@ -233,6 +266,8 @@ func constI(n int) func(int32) int { return func(int32) int { return n } } // CPython: Include/internal/pycore_opcode_metadata.h // _PyOpcode_num_popped / _PyOpcode_num_pushed switches. var opcodeArity = map[Opcode]arityEntry{ + CACHE: arity(0, 0), + EXTENDED_ARG: arity(0, 0), NOP: arity(0, 0), RESUME: arity(0, 0), POP_TOP: arity(1, 0), diff --git a/compile/instrseq.go b/compile/instrseq.go index c339c2107..066d54454 100644 --- a/compile/instrseq.go +++ b/compile/instrseq.go @@ -149,11 +149,28 @@ func (s *Sequence) Addop(op Opcode, oparg int32, loc ast.Pos) { s.Instrs = append(s.Instrs, Instr{ Op: op, Oparg: oparg, - Loc: loc, + Loc: normalizeLoc(loc), Handler: ExceptHandlerInfo{Label: -1}, }) } +// normalizeLoc maps the zero-value ast.Pos that codegen passes for an +// artificial instruction onto NO_LOCATION (lineno -1). CPython spells +// these ADDOP(c, NO_LOCATION, ...); gopy's codegen idiom is ast.Pos{}. +// A real source span always has lineno >= 1, and the module RESUME +// carries lineno 0 with end_lineno 1, so a fully-zero Pos is +// unambiguously the artificial sentinel. Keeping it as lineno 0 would +// serialize a bogus (0, 0, 0, 0) span and defeat the flowgraph +// propagate_line_numbers pass, which only fills lineno < 0 holes. +// +// CPython: Python/flowgraph.c:50 NO_LOCATION +func normalizeLoc(loc ast.Pos) ast.Pos { + if loc == (ast.Pos{}) { + return noLocation + } + return loc +} + // Insert inserts at pos and shifts following entries right by one. // Any previously-bound label that pointed at pos or later is bumped // up by one to preserve its target. diff --git a/compile/opcodes_gen.go b/compile/opcodes_gen.go index d42b0f030..3b792e514 100644 --- a/compile/opcodes_gen.go +++ b/compile/opcodes_gen.go @@ -387,34 +387,50 @@ var opcodeFlags = [267]uint16{ UNPACK_SEQUENCE: 0x0301, YIELD_VALUE: 0x0001, RESUME: 0x4341, - BINARY_OP_ADD_FLOAT: 0x0500, - BINARY_OP_ADD_INT: 0x0700, - BINARY_OP_ADD_UNICODE: 0x0500, - BINARY_OP_EXTEND: 0x0380, - BINARY_OP_MULTIPLY_FLOAT: 0x0500, - BINARY_OP_MULTIPLY_INT: 0x0700, - BINARY_OP_SUBSCR_DICT: 0x0700, - BINARY_OP_SUBSCR_GETITEM: 0x0080, - BINARY_OP_SUBSCR_LIST_INT: 0x0680, - BINARY_OP_SUBSCR_LIST_SLICE: 0x0700, - BINARY_OP_SUBSCR_STR_INT: 0x0680, - BINARY_OP_SUBSCR_TUPLE_INT: 0x0480, - BINARY_OP_SUBTRACT_FLOAT: 0x0500, - BINARY_OP_SUBTRACT_INT: 0x0700, - CALL_ALLOC_AND_ENTER_INIT: 0x4381, - CALL_BOUND_METHOD_EXACT_ARGS: 0x0681, - CALL_BOUND_METHOD_GENERAL: 0x4781, - CALL_BUILTIN_CLASS: 0x03c1, - CALL_BUILTIN_FAST: 0x03c1, - CALL_BUILTIN_FAST_WITH_KEYWORDS: 0x03c1, - CALL_BUILTIN_O: 0x0741, - CALL_ISINSTANCE: 0x4381, - CALL_KW_BOUND_METHOD: 0x0781, - CALL_KW_NON_PY: 0x0741, - CALL_KW_PY: 0x0781, - CALL_LEN: 0x4380, - CALL_LIST_APPEND: 0x0381, - CALL_METHOD_DESCRIPTOR_FAST: 0x0741, + // Pseudo-instruction flags. The legacy opcodes_go generator stopped at + // the real-opcode range and never emitted these, so HasArg/HasJump + // returned false for every pseudo op (breaking dis.hasarg widths and + // the has_arg surface). Transcribed verbatim from CPython's metadata. + // + // CPython: Include/internal/pycore_opcode_metadata.h _PyOpcode_opcode_metadata + JUMP: 0x0349, // HAS_ARG|HAS_JUMP|HAS_EVAL_BREAK|HAS_ERROR|HAS_ESCAPES + JUMP_IF_FALSE: 0x0309, // HAS_ARG|HAS_JUMP|HAS_ERROR|HAS_ESCAPES + JUMP_IF_TRUE: 0x0309, // HAS_ARG|HAS_JUMP|HAS_ERROR|HAS_ESCAPES + JUMP_NO_INTERRUPT: 0x0009, // HAS_ARG|HAS_JUMP + LOAD_CLOSURE: 0x0821, // HAS_ARG|HAS_LOCAL|HAS_PURE + POP_BLOCK: 0x0800, // HAS_PURE + SETUP_CLEANUP: 0x0801, // HAS_PURE|HAS_ARG + SETUP_FINALLY: 0x0801, // HAS_PURE|HAS_ARG + SETUP_WITH: 0x0801, // HAS_PURE|HAS_ARG + STORE_FAST_MAYBE_NULL: 0x0221, // HAS_ARG|HAS_LOCAL|HAS_ESCAPES + BINARY_OP_ADD_FLOAT: 0x0500, + BINARY_OP_ADD_INT: 0x0700, + BINARY_OP_ADD_UNICODE: 0x0500, + BINARY_OP_EXTEND: 0x0380, + BINARY_OP_MULTIPLY_FLOAT: 0x0500, + BINARY_OP_MULTIPLY_INT: 0x0700, + BINARY_OP_SUBSCR_DICT: 0x0700, + BINARY_OP_SUBSCR_GETITEM: 0x0080, + BINARY_OP_SUBSCR_LIST_INT: 0x0680, + BINARY_OP_SUBSCR_LIST_SLICE: 0x0700, + BINARY_OP_SUBSCR_STR_INT: 0x0680, + BINARY_OP_SUBSCR_TUPLE_INT: 0x0480, + BINARY_OP_SUBTRACT_FLOAT: 0x0500, + BINARY_OP_SUBTRACT_INT: 0x0700, + CALL_ALLOC_AND_ENTER_INIT: 0x4381, + CALL_BOUND_METHOD_EXACT_ARGS: 0x0681, + CALL_BOUND_METHOD_GENERAL: 0x4781, + CALL_BUILTIN_CLASS: 0x03c1, + CALL_BUILTIN_FAST: 0x03c1, + CALL_BUILTIN_FAST_WITH_KEYWORDS: 0x03c1, + CALL_BUILTIN_O: 0x0741, + CALL_ISINSTANCE: 0x4381, + CALL_KW_BOUND_METHOD: 0x0781, + CALL_KW_NON_PY: 0x0741, + CALL_KW_PY: 0x0781, + CALL_LEN: 0x4380, + CALL_LIST_APPEND: 0x0381, + CALL_METHOD_DESCRIPTOR_FAST: 0x0741, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: 0x0741, CALL_METHOD_DESCRIPTOR_NOARGS: 0x0741, CALL_METHOD_DESCRIPTOR_O: 0x0741, diff --git a/compile/parity_dump.go b/compile/parity_dump.go index 881db13cc..4e3ea7f9b 100644 --- a/compile/parity_dump.go +++ b/compile/parity_dump.go @@ -283,6 +283,9 @@ func formatConstRepr(v any) string { return pyReprBytes(x) case *ConstTuple: return formatTupleRepr(x.Values) + case *ConstSlice: + return "slice(" + formatConstRepr(x.Start) + ", " + + formatConstRepr(x.Stop) + ", " + formatConstRepr(x.Step) + ")" case *Unit: return fmt.Sprintf("", x.Qualname, x.FirstLineno) case *Code: diff --git a/compile/testbridge.go b/compile/testbridge.go new file mode 100644 index 000000000..2ae2d9b79 --- /dev/null +++ b/compile/testbridge.go @@ -0,0 +1,152 @@ +// Bridge entry points for the _testinternalcapi compiler-pipeline +// helpers (new_instruction_sequence, compiler_codegen, optimize_cfg, +// assemble_code_object). They expose the codegen / cfg / assemble +// passes to module/_testinternalcapi so the vendored test harness in +// test/support/bytecode_helper.py and the test_peepholer / +// test_compiler_assemble suites can drive each stage directly. +// +// CPython: Modules/_testinternalcapi.c:713 new_instruction_sequence ff. + +package compile + +import ( + "fmt" + + "github.com/tamnd/gopy/ast" + "github.com/tamnd/gopy/future" + "github.com/tamnd/gopy/symtable" +) + +// NewSequence allocates an empty instruction sequence. Mirrors +// _PyInstructionSequence_New, which _testinternalcapi exposes as +// new_instruction_sequence. +// +// CPython: Python/instruction_sequence.c:38 _PyInstructionSequence_New +func NewSequence() *Sequence { + return &Sequence{} +} + +// AddopRaw appends an instruction with the location stored verbatim, +// bypassing normalizeLoc. The InstructionSequence Python object hands +// out whatever six-tuple round-trips through get_instructions, so a +// fully-zero (1,0,0,0)-style span must survive unchanged rather than +// collapse onto NO_LOCATION. +// +// CPython: Python/instruction_sequence.c:116 _PyInstructionSequence_Addop +func (s *Sequence) AddopRaw(op Opcode, oparg int32, lineno, colOffset, endLineno, endColOffset int) error { + if op < 0 || op > MaxOpcode { + return fmt.Errorf("compile: opcode out of range") + } + if oparg < 0 || oparg >= MaxOparg { + return fmt.Errorf("compile: oparg out of range") + } + s.Instrs = append(s.Instrs, Instr{ + Op: op, + Oparg: oparg, + Loc: ast.Pos{ + Lineno: lineno, + ColOffset: colOffset, + EndLineno: endLineno, + EndColOffset: endColOffset, + }, + Handler: ExceptHandlerInfo{Label: -1}, + }) + return nil +} + +// NewLabelID allocates a fresh label and returns its integer id, the +// value InstructionSequence.new_label hands to Python. +// +// CPython: Python/instruction_sequence.c:58 _PyInstructionSequence_NewLabel +func (s *Sequence) NewLabelID() int { + return s.NewLabel().ID() +} + +// UseLabelID binds the label id to the next instruction position. +// +// CPython: Python/instruction_sequence.c:65 _PyInstructionSequence_UseLabel +func (s *Sequence) UseLabelID(id int) { + s.UseLabel(JumpTargetLabel{id: id}) +} + +// ApplyJumpLabels resolves every jump oparg from a label id into the +// bound instruction offset. Idempotent (see ApplyLabelMap). +// +// CPython: Python/instruction_sequence.c get_instructions calls +// instr_sequence_apply_jumps before building the tuples. +func (s *Sequence) ApplyJumpLabels() { + s.ApplyLabelMap(hasJumpTarget) +} + +// CodeGenForTest runs the front-end (future + preprocess + symtable + +// Codegen) and returns the post-codegen Unit, the same value +// _PyCompile_CodeGen builds before handing back (instr_seq, metadata). +// saveNestedSeqs is set so the returned Unit's Seq carries the nested +// child sequences get_nested walks. gopy's Codegen already emits the +// trailing implicit return, so we do NOT call AddReturnAtEnd again. +// +// CPython: Python/compile.c:1607 _PyCompile_CodeGen +func CodeGenForTest(mod ast.Mod, filename string, optimize int) (*Unit, error) { + ff, err := future.FromAST(mod, filename) + if err != nil { + return nil, err + } + ast.Preprocess(mod, ast.PreprocessOptions{ + Filename: filename, + OptimizeLevel: optimize, + FFFeatures: ff.Bits, + EnableWarnings: true, + }) + st, err := symtable.Build(mod, filename, ff) + if err != nil { + return nil, err + } + c := NewCompiler(filename, optimize, ff, st) + c.saveNestedSeqs = true + scope := c.Symtable.Top + if scope == nil { + return nil, fmt.Errorf("compile: symtable has no top entry") + } + return c.Codegen(scope, mod) +} + +// OptimizeCfgForTest mirrors _PyCompile_OptimizeCfg: build a cfg from +// seq, run the optimize pass with nparams=0 firstlineno=1 (the +// optimize_cfg entry's fixed arguments), recompute stack depth, +// optimize LOAD_FAST, and convert back to an instruction sequence. +// The consts slice is mutated in place so the caller can read the +// optimized const table back out. +// +// CPython: Python/flowgraph.c:4126 _PyCompile_OptimizeCfg +func OptimizeCfgForTest(seq *Sequence, consts *[]any, nlocals int) (*Sequence, error) { + g := cfgFromSequence(seq) + if err := cfgOptimizeCodeUnit(g, consts, nlocals, 0, 1); err != nil { + return nil, err + } + if _, err := cfgCalculateStackdepth(g); err != nil { + return nil, err + } + if err := optimizeLoadFast(g); err != nil { + return nil, err + } + out := &Sequence{} + cfgToSequence(g, out) + return out, nil +} + +// AssembleForTest mirrors _PyCompile_Assemble: build a cfg from seq, +// translate jump labels to targets, convert the optimized cfg back +// into an instruction sequence (recomputing stack depth and the +// localsplus layout), and assemble the code object. +// +// CPython: Python/compile.c:1657 _PyCompile_Assemble +func AssembleForTest(unit *Unit, seq *Sequence, filename string) (*Code, error) { + g := cfgFromSequence(seq) + cfgTranslateJumpLabelsToTargets(g) + optimized := &Sequence{} + stackdepth, nlocalsplus, err := cfgOptimizedCfgToInstructionSequence(g, unit, 0, optimized) + if err != nil { + return nil, err + } + return assembleMakeCodeObject(unit, unit.Consts, stackdepth, optimized, nlocalsplus, 0, filename), nil +} diff --git a/compile/warn.go b/compile/warn.go index ebab2ee8c..5d2eede04 100644 --- a/compile/warn.go +++ b/compile/warn.go @@ -261,6 +261,12 @@ func (c *Compiler) checkIndex(obj ast.Expr, idx ast.Expr) error { // // CPython: Python/compile.c:237 _PyCompile_Warn func (c *Compiler) warnAt(pos ast.Pos, format string, args ...any) error { + // Suppressed while compiling the exception-path copy of a finally + // body, so a warning there is not emitted a second time. + // CPython: Python/compile.c:1213 _PyCompile_Warn (c_disable_warning) + if c.disableWarning > 0 { + return nil + } if WarnHook == nil { return nil } diff --git a/format/format.go b/format/format.go index c328b691d..665aeba3a 100644 --- a/format/format.go +++ b/format/format.go @@ -15,7 +15,6 @@ import ( "fmt" "math" "math/big" - "strconv" "strings" "unicode/utf8" @@ -26,6 +25,36 @@ import ( // grammar. var ErrInvalidSpec = errors.New("format: invalid format specifier") +// ErrInvalidSpecifier is returned by ParseSpec when more than one +// character remains for the type field, i.e. the spec is unparseable. +// CPython raises this with the offending object's type name appended, +// so callers (which hold the object) wrap it into the full +// "Invalid format specifier '' for object of type ''" +// ValueError. +// +// CPython: Python/formatter_unicode.c:305 (end-pos > 1 branch) +var ErrInvalidSpecifier = errors.New("format: invalid format specifier (trailing characters)") + +// ErrTooManyDigits mirrors the "Too many decimal digits in format +// string" ValueError get_integer raises when a width or precision digit +// run overflows the platform integer. +// +// CPython: Python/formatter_unicode.c:78 Too many decimal digits +var ErrTooManyDigits = errors.New("Too many decimal digits in format string") //nolint:staticcheck // Mirror CPython error text. + +// ErrPrecisionTooBig mirrors the "precision too big" ValueError raised +// when the requested float/complex precision exceeds INT_MAX. +// +// CPython: Python/formatter_unicode.c:1166 precision too big +var ErrPrecisionTooBig = errors.New("precision too big") //nolint:staticcheck // Mirror CPython error text. + +// ErrMissingPrecision mirrors the "Format specifier missing precision" +// ValueError raised when a '.' is not followed by a precision or a +// fractional grouping separator. +// +// CPython: Python/formatter_unicode.c:296 Format specifier missing precision +var ErrMissingPrecision = errors.New("Format specifier missing precision") //nolint:staticcheck // Mirror CPython error text. + // errCommaAndUnderscore mirrors invalid_comma_and_underscore from // formatter_unicode.c: the user requested both grouping styles in the // same spec. @@ -86,45 +115,52 @@ type Spec struct { Type byte // 0 if unspecified } -func isAlignToken(c byte) bool { +func isAlignToken(c rune) bool { return c == '<' || c == '>' || c == '=' || c == '^' } -func isSignToken(c byte) bool { +func isSignToken(c rune) bool { return c == '+' || c == '-' || c == ' ' } // ParseSpec mirrors parse_internal_render_format_spec. // +// The spec is decoded to code points up front: every grammar element +// except the fill character is ASCII, but the fill character may be any +// code point (e.g. a multi-byte alignment fill like "🖤>6"), so the +// parser must index by rune, not byte, exactly like CPython's +// READ_spec over a PyUnicode buffer. +// // CPython: Python/formatter_unicode.c:L150 parse_internal_render_format_spec func ParseSpec(s string) (Spec, error) { spec := Spec{Fill: -1, Width: -1, Precision: -1} + r := []rune(s) i := 0 // [[fill]align] - if len(s)-i >= 2 && isAlignToken(s[i+1]) { - spec.Fill = rune(s[i]) - spec.Align = s[i+1] + if len(r)-i >= 2 && isAlignToken(r[i+1]) { + spec.Fill = r[i] + spec.Align = byte(r[i+1]) i += 2 - } else if len(s)-i >= 1 && isAlignToken(s[i]) { - spec.Align = s[i] + } else if len(r)-i >= 1 && isAlignToken(r[i]) { + spec.Align = byte(r[i]) i++ } // [sign] - if i < len(s) && isSignToken(s[i]) { - spec.Sign = s[i] + if i < len(r) && isSignToken(r[i]) { + spec.Sign = byte(r[i]) i++ } // [z] - coerce -0.0 to 0.0 for floats - if i < len(s) && s[i] == 'z' { + if i < len(r) && r[i] == 'z' { spec.NoNegZero = true i++ } // [#] - if i < len(s) && s[i] == '#' { + if i < len(r) && r[i] == '#' { spec.Alt = true i++ } @@ -132,7 +168,7 @@ func ParseSpec(s string) (Spec, error) { // [0] zero-pad shortcut // CPython: Python/formatter_unicode.c:213 !fill_char_specified // Always sets fill='0'; only sets align='=' if align was not explicit. - if i < len(s) && s[i] == '0' && spec.Fill == -1 { + if i < len(r) && r[i] == '0' && spec.Fill == -1 { spec.Fill = '0' if spec.Align == 0 { spec.Zero = true @@ -142,7 +178,7 @@ func ParseSpec(s string) (Spec, error) { } // [width] - width, consumed, err := parseInt(s, i) + width, consumed, err := parseInt(r, i) if err != nil { return spec, err } @@ -156,18 +192,18 @@ func ParseSpec(s string) (Spec, error) { // messages can fire. // // CPython: Python/formatter_unicode.c:236 grouping section - if i < len(s) && s[i] == ',' { + if i < len(r) && r[i] == ',' { spec.Thousands = ',' i++ } - if i < len(s) && s[i] == '_' { + if i < len(r) && r[i] == '_' { if spec.Thousands != 0 { return spec, errCommaAndUnderscore } spec.Thousands = '_' i++ } - if i < len(s) && s[i] == ',' { + if i < len(r) && r[i] == ',' { if spec.Thousands == '_' { return spec, errCommaAndUnderscore } @@ -179,9 +215,9 @@ func ParseSpec(s string) (Spec, error) { // [.precision][frac_thousands] // CPython: Python/formatter_unicode.c:257 Parse field precision // CPython 3.14: Python/formatter_unicode.c:265 frac_thousands_separator - if i < len(s) && s[i] == '.' { + if i < len(r) && r[i] == '.' { i++ - prec, consumed, err := parseInt(s, i) + prec, consumed, err := parseInt(r, i) if err != nil { return spec, err } @@ -189,15 +225,23 @@ func ParseSpec(s string) (Spec, error) { spec.Precision = prec i += consumed } - // Optional frac thousands separator after the (optional) precision. - if i < len(s) && s[i] == ',' { + // Optional frac thousands separator after the (optional) + // precision. CPython parses the comma and the underscore in two + // separate `if` blocks (not else-if) so a "comma then + // underscore" run (e.g. ".,_f") hits the + // invalid_comma_and_underscore branch instead of silently + // taking only the comma. + // + // CPython: Python/formatter_unicode.c:266 frac comma/underscore + if i < len(r) && r[i] == ',' { if consumed == 0 { spec.Precision = -1 } spec.FracThousands = ',' i++ consumed++ - } else if i < len(s) && s[i] == '_' { + } + if i < len(r) && r[i] == '_' { if spec.FracThousands != 0 { return spec, errCommaAndUnderscore } @@ -209,22 +253,28 @@ func ParseSpec(s string) (Spec, error) { consumed++ } // Trailing comma after underscore → error - if i < len(s) && s[i] == ',' && spec.FracThousands == '_' { + if i < len(r) && r[i] == ',' && spec.FracThousands == '_' { return spec, errCommaAndUnderscore } if consumed == 0 { - return spec, ErrInvalidSpec + return spec, ErrMissingPrecision } } // [type] - if i < len(s) { - spec.Type = s[i] + if i < len(r) { + spec.Type = byte(r[i]) i++ } - if i != len(s) { - return spec, ErrInvalidSpec + // More than one character remains for the type field: the spec is + // invalid. CPython raises "Invalid format specifier '' for + // object of type ''"; ParseSpec is type-agnostic, so it + // returns the sentinel and the caller appends the type name. + // + // CPython: Python/formatter_unicode.c:305 (end-pos > 1 branch) + if i != len(r) { + return spec, ErrInvalidSpecifier } if spec.Type != 0 { if err := validateThousands(spec.Thousands, spec.Type); err != nil { @@ -238,19 +288,28 @@ func ParseSpec(s string) (Spec, error) { return spec, nil } -func parseInt(s string, i int) (val, consumed int, err error) { +// parseInt ports get_integer: it consumes a run of decimal digits and +// accumulates them as a (signed) integer, detecting overflow before it +// happens. Overflow past the platform word size raises "Too many +// decimal digits in format string" rather than a generic invalid-spec +// error, matching CPython's PY_SSIZE_T_MAX guard. +// +// CPython: Python/formatter_unicode.c:61 get_integer +func parseInt(r []rune, i int) (val, consumed int, err error) { start := i - for i < len(s) && s[i] >= '0' && s[i] <= '9' { + acc := 0 + for i < len(r) && r[i] >= '0' && r[i] <= '9' { + d := int(r[i] - '0') + if acc > (math.MaxInt-d)/10 { + return 0, i - start, ErrTooManyDigits + } + acc = acc*10 + d i++ } if i == start { return 0, 0, nil } - v, err := strconv.Atoi(s[start:i]) - if err != nil { - return 0, 0, ErrInvalidSpec - } - return v, i - start, nil + return acc, i - start, nil } // FormatString renders s under spec. Mirrors format_string_internal. @@ -264,6 +323,11 @@ func FormatString(s string, spec Spec) (string, error) { if err := validateThousands(spec.Thousands, t); err != nil { return "", err } + // CPython: Python/formatter_unicode.c:888 negative-0 coercion is not + // allowed on strings. + if spec.NoNegZero { + return "", errors.New("Negative zero coercion (z) not allowed in string format specifier") //nolint:staticcheck // Mirror CPython error text. + } if spec.Sign == ' ' { return "", fmt.Errorf("ValueError: Space not allowed in string format specifier") } @@ -541,6 +605,14 @@ func FormatFloat(v float64, spec Spec) (string, error) { return "", err } + // A precision wider than a C int cannot be honoured by the digit + // generator. CPython rejects it up front with "precision too big". + // + // CPython: Python/formatter_unicode.c:1165 precision > INT_MAX + if spec.Precision > math.MaxInt32 { + return "", ErrPrecisionTooBig + } + precision := spec.Precision flags := pystrconv.FloatFormatFlag(0) switch spec.Sign { diff --git a/frame/frame.go b/frame/frame.go index c2ef827c7..e748a695b 100644 --- a/frame/frame.go +++ b/frame/frame.go @@ -60,6 +60,19 @@ type Frame struct { InstrPtr int PrevInstr int + // LinenoJumped is set when the f_lineno setter (the debugger line + // jump) relocates InstrPtr during a trace callback. The dispatch + // loop consults it after the INSTRUMENTED_LINE event to resume at + // the new target instead of running the hidden opcode, then clears + // it. A bare InstrPtr-before/after comparison would also trip on the + // EXTENDED_ARG-prefix advance fetchExtended performs, so the jump + // needs its own explicit signal. + // + // CPython: Objects/frameobject.c:1640 frame_lineno_set_impl sets + // frame->instr_ptr, which Python/bytecodes.c INSTRUMENTED_LINE then + // detects via instr_ptr != this_instr. + LinenoJumped bool + // StackTop is the index into LocalsPlus where the live value // stack ends. The stack starts at StackBase. StackTop int @@ -500,6 +513,13 @@ func (f *Frame) FrameFastLocal(i int) objects.Object { if f.snapshot != nil && i < len(f.snapshot) { return f.snapshot[i].AsObject() } + // The cycle collector can traverse a frame whose LocalsPlus has not + // yet been grown to its full width (an opcode that builds a container + // can trip auto-GC before the slot array is sized). Guard the index so + // traversal sees an unbound slot rather than panicking. + if i >= len(f.LocalsPlus) { + return nil + } return f.LocalsPlus[i].AsObject() } @@ -513,6 +533,9 @@ func (f *Frame) FrameCellLocal(i int) objects.Object { if f.snapshot != nil && idx < len(f.snapshot) { return f.snapshot[idx].AsObject() } + if idx >= len(f.LocalsPlus) { + return nil + } return f.LocalsPlus[idx].AsObject() } @@ -526,6 +549,9 @@ func (f *Frame) FrameFreeLocal(i int) objects.Object { if f.snapshot != nil && idx < len(f.snapshot) { return f.snapshot[idx].AsObject() } + if idx >= len(f.LocalsPlus) { + return nil + } return f.LocalsPlus[idx].AsObject() } diff --git a/gopy_audit b/gopy_audit new file mode 100755 index 000000000..1fac14718 Binary files /dev/null and b/gopy_audit differ diff --git a/gopy_base b/gopy_base new file mode 100755 index 000000000..357bb14fa Binary files /dev/null and b/gopy_base differ diff --git a/gopy_baseline b/gopy_baseline new file mode 100755 index 000000000..357bb14fa Binary files /dev/null and b/gopy_baseline differ diff --git a/gopy_dbg b/gopy_dbg new file mode 100755 index 000000000..9250d1b11 Binary files /dev/null and b/gopy_dbg differ diff --git a/marshal/marshal.go b/marshal/marshal.go index 9813d4e2e..7e85d7b01 100644 --- a/marshal/marshal.go +++ b/marshal/marshal.go @@ -63,6 +63,7 @@ const ( typeFrozenset = '>' typeComplex = 'x' typeBinaryComplex = 'y' + typeSlice = ':' ) // flagRef is OR'd onto a type tag to signal that the object should be @@ -214,6 +215,7 @@ const ( refKindSmallInt refKindEmptyTuple refKindCode + refKindSlice ) // refKey is a comparable identity for the marshal refs table. @@ -282,6 +284,11 @@ func (e *encoder) refKeyFor(v any) (refKey, bool) { return refKey{}, false } return refKey{kind: refKindCode, p: uintptr(unsafe.Pointer(x))}, true + case *objects.Slice: + // CPython forces a ref reservation for every slice (w_object's + // `PyCode_Check(v) || PySlice_Check(v)` branch), so a slice always + // goes through the memo keyed on its pointer identity. + return refKey{kind: refKindSlice, p: uintptr(unsafe.Pointer(x))}, true } return refKey{}, false } @@ -423,10 +430,57 @@ func (e *encoder) writeBody(v any, flag byte) error { return nil case []any: return e.writeTuple(x, flag) + case ast.FrozenSet: + return e.writeFrozenSetConst(x, flag) + case *objects.Slice: + return e.writeSlice(x, flag) } return fmt.Errorf("%w: %T", ErrUnmarshallable, v) } +// writeSlice encodes a slice constant as TYPE_SLICE followed by its +// start, stop, and step. Each bound is converted to a plain marshal +// value so it dispatches back through write. +// +// CPython: Python/marshal.c:719 w_object PySlice_Check arm +func (e *encoder) writeSlice(s *objects.Slice, flag byte) error { + if err := e.writeByte(typeSlice | flag); err != nil { + return err + } + for _, bound := range []objects.Object{s.Start, s.Stop, s.Step} { + v, err := fromObject(bound) + if err != nil { + return err + } + if err := e.write(v); err != nil { + return err + } + } + return nil +} + +// writeFrozenSetConst encodes the compiler's ast.FrozenSet constant as +// TYPE_FROZENSET. The compiler folds a membership test against a set +// display (`x in {1, 2, 3}`) into a frozenset constant; its items are +// already plain marshal values, so they dispatch straight back through +// write. Counterpart to writeSet, which handles a live *objects.Set. +// +// CPython: Python/marshal.c w_object PyFrozenSet_Type +func (e *encoder) writeFrozenSetConst(items ast.FrozenSet, flag byte) error { + if err := e.writeByte(typeFrozenset | flag); err != nil { + return err + } + if err := e.writeInt32(int32(len(items))); err != nil { + return err + } + for _, item := range items { + if err := e.write(item); err != nil { + return err + } + } + return nil +} + // writeTuple emits a tuple body. CPython picks TYPE_SMALL_TUPLE for // length < 256 and TYPE_TUPLE otherwise. // @@ -764,6 +818,8 @@ func (d *decoder) decodeTag(tag byte) (any, error) { return d.readComplexText() case typeBinaryComplex: return d.readBinaryComplex() + case typeSlice: + return d.readSlice() case typeRef: n, err := d.readInt32() if err != nil { @@ -777,6 +833,39 @@ func (d *decoder) decodeTag(tag byte) (any, error) { return nil, fmt.Errorf("marshal: unknown type tag %q (0x%02x)", rune(tag), tag) } +// readSlice decodes a TYPE_SLICE: three marshal objects (start, stop, +// step) reassembled into a runtime slice object. Each bound is lifted +// through toObject since the slice fields hold Objects. +// +// CPython: Python/marshal.c:1691 r_object TYPE_SLICE +func (d *decoder) readSlice() (any, error) { + start, err := d.read() + if err != nil { + return nil, err + } + stop, err := d.read() + if err != nil { + return nil, err + } + step, err := d.read() + if err != nil { + return nil, err + } + startO, err := toObject(start) + if err != nil { + return nil, err + } + stopO, err := toObject(stop) + if err != nil { + return nil, err + } + stepO, err := toObject(step) + if err != nil { + return nil, err + } + return objects.NewSlice(startO, stopO, stepO), nil +} + func (d *decoder) readTuple(n int) ([]any, error) { out := make([]any, n) for i := 0; i < n; i++ { @@ -899,6 +988,35 @@ func toObject(v any) (objects.Object, error) { return objects.NewStr(x), nil case ast.EllipsisType: return objects.Ellipsis(), nil + case []byte: + return objects.NewBytes(x), nil + case complex128: + return objects.NewComplex(real(x), imag(x)), nil + case *big.Int: + return objects.NewIntFromBig(x), nil + case []any: + // A tuple element of a set/frozenset: convert each item and + // build an immutable tuple, the only sequence form that can be + // a set key. + items := make([]objects.Object, len(x)) + for i, raw := range x { + item, err := toObject(raw) + if err != nil { + return nil, err + } + items[i] = item + } + return objects.NewTuple(items), nil + case ast.FrozenSet: + items := make([]objects.Object, len(x)) + for i, raw := range x { + item, err := toObject(raw) + if err != nil { + return nil, err + } + items[i] = item + } + return objects.NewFrozenset(items) case objects.Object: return x, nil } @@ -959,6 +1077,9 @@ func fromObject(obj objects.Object) (any, error) { if obj.Type() == objects.NoneType() { return nil, nil } + if obj == objects.Ellipsis() { + return ast.EllipsisType{}, nil + } if obj.Type().Str != nil { s, err := obj.Type().Str(obj) if err == nil { diff --git a/module/_collections/module.go b/module/_collections/module.go index 9eab9fd8a..de555aad9 100644 --- a/module/_collections/module.go +++ b/module/_collections/module.go @@ -1899,7 +1899,15 @@ func tupleGetterDescrGet(descr objects.Object, owner objects.Object, _ *objects. if tg.Index < 0 || tg.Index >= tup.Len() { return nil, fmt.Errorf("IndexError: tuple index out of range") } - return tup.Item(tg.Index), nil + // Hand back an owned reference. PyTuple_GET_ITEM borrows, so the + // descriptor must incref before returning, otherwise the VM decrefs the + // field as a temporary and over-releases the value still stored in the + // tuple. + // + // CPython: Modules/_collectionsmodule.c:2682 tuplegetter_descr_get + result := tup.Item(tg.Index) + objects.Incref(result) + return result, nil } // tupleGetterDescrSet rejects writes and deletes. diff --git a/module/_opcode/module.go b/module/_opcode/module.go index 5c8b2b0da..6732a74ac 100644 --- a/module/_opcode/module.go +++ b/module/_opcode/module.go @@ -145,20 +145,61 @@ func opcodeArg(name string, args []objects.Object, kwargs map[string]objects.Obj // // CPython: Include/internal/pycore_opcode_metadata.h IS_VALID_OPCODE func isValidOpcode(op int) bool { - if op < 0 || op > 255 { + if op < 0 || op > 266 { return false } return compile.Opcode(op).Name() != "" } -// stackEffect would call PyCompile_OpcodeStackEffectWithJump; gopy does -// not have an analytic stack-effect table yet, so we raise so callers -// learn early. dis.stack_effect is the only consumer and never fires -// during opcode.py / dis.py module load. +// stackEffect implements stack_effect(opcode, oparg=None, *, jump=None), +// returning the net operand-stack effect of opcode. jump selects the +// branch column for jumping/exception opcodes (None/True/False). // // CPython: Modules/_opcode.c:37 _opcode_stack_effect_impl func stackEffect(args []objects.Object, kwargs map[string]objects.Object) (objects.Object, error) { - return nil, fmt.Errorf("NotImplementedError: _opcode.stack_effect is not yet ported") + if len(args) < 1 || len(args) > 2 { + return nil, fmt.Errorf("TypeError: stack_effect() takes 1 or 2 positional arguments") + } + opObj, ok := args[0].(*objects.Int) + if !ok { + return nil, fmt.Errorf("TypeError: stack_effect() argument 'opcode' must be int") + } + opv, ok := opObj.Int64() + if !ok { + return nil, fmt.Errorf("OverflowError: opcode value too large") + } + + oparg := int32(0) + if len(args) == 2 && args[1] != objects.None() { + argObj, ok := args[1].(*objects.Int) + if !ok { + return nil, fmt.Errorf("TypeError: stack_effect() argument 'oparg' must be int or None") + } + v, ok := argObj.Int64() + if !ok { + return nil, fmt.Errorf("OverflowError: oparg value too large") + } + oparg = int32(v) + } + + // jump: None -> -1, True -> 1, False -> 0. + jump := -1 + switch kwargs["jump"] { + case nil, objects.None(): + jump = -1 + case objects.True(): + jump = 1 + case objects.False(): + jump = 0 + default: + return nil, fmt.Errorf("ValueError: stack_effect: jump must be False, True or None") + } + + effect := compile.OpcodeStackEffectWithJump(int(opv), oparg, jump) + if effect == compile.InvalidStackEffect { + return nil, fmt.Errorf("ValueError: invalid opcode or oparg") + } + return objects.NewInt(int64(effect)), nil } // CPython: Modules/_opcode.c:83 _opcode_is_valid_impl diff --git a/module/_testcapi/codenewempty.go b/module/_testcapi/codenewempty.go new file mode 100644 index 000000000..9c921514e --- /dev/null +++ b/module/_testcapi/codenewempty.go @@ -0,0 +1,68 @@ +// Port of CPython's PyCode_NewEmpty, exposed to the test suite as +// _testcapi.code_newempty. test_code.py drives it to build a minimal +// code object that raises AssertionError on exec. +// +// CPython: Objects/codeobject.c:955 PyCode_NewEmpty + +package testcapi + +import ( + "fmt" + + "github.com/tamnd/gopy/objects" +) + +// assert0 is the six-byte bytecode PyCode_NewEmpty installs: resume at +// function start, load the AssertionError common constant, then raise +// it with one argument count. +// +// CPython: Objects/codeobject.c:941 assert0 +var assert0 = []byte{ + 128, 0, // RESUME, RESUME_AT_FUNC_START + 81, 0, // LOAD_COMMON_CONSTANT, CONSTANT_ASSERTIONERROR + 104, 1, // RAISE_VARARGS, 1 +} + +// emptyLinetable is the two-byte location table covering the three code +// units of assert0 with no column info and a zero line offset. +// +// CPython: Objects/codeobject.c:947 linetable +// +// (1<<7) | (PY_CODE_LOCATION_INFO_NO_COLUMNS<<3) | (3-1) = 0xEA +var emptyLinetable = []byte{0xEA, 0} + +// codeNewempty ports _testcapi.code_newempty(filename, funcname, +// firstlineno), a thin wrapper over PyCode_NewEmpty. +// +// CPython: Modules/_testcapimodule.c code_newempty / Objects/codeobject.c:955 +func codeNewempty(args []objects.Object, _ map[string]objects.Object) (objects.Object, error) { + if len(args) != 3 { + return nil, fmt.Errorf("TypeError: code_newempty expected 3 arguments, got %d", len(args)) + } + filename, ok := args[0].(*objects.Unicode) + if !ok { + return nil, fmt.Errorf("TypeError: argument 1 must be str, not %s", args[0].Type().Name) + } + funcname, ok := args[1].(*objects.Unicode) + if !ok { + return nil, fmt.Errorf("TypeError: argument 2 must be str, not %s", args[1].Type().Name) + } + lineno, ok := args[2].(*objects.Int) + if !ok { + return nil, fmt.Errorf("TypeError: argument 3 must be int, not %s", args[2].Type().Name) + } + firstlineno, _ := lineno.Int64() + + c := objects.NewCode() + c.Filename = filename.Value() + c.Name = funcname.Value() + c.Qualname = funcname.Value() + c.Code = append([]byte(nil), assert0...) + c.Firstlineno = int(firstlineno) + c.Linetable = append([]byte(nil), emptyLinetable...) + c.Stacksize = 1 + c.SyncConstObjs() + c.SyncNameObjs() + c.SyncLocalsplusCounts() + return c, nil +} diff --git a/module/_testcapi/module.go b/module/_testcapi/module.go index f54c84a48..ecbd97c3f 100644 --- a/module/_testcapi/module.go +++ b/module/_testcapi/module.go @@ -246,6 +246,7 @@ func buildModule() (*objects.Module, error) { {"config_getint", configGetint}, {"config_names", configNames}, {"run_in_subinterp", runInSubinterp}, + {"code_newempty", codeNewempty}, } for _, w := range wrappers { if err := d.SetItem(objects.NewStr(w.name), objects.NewBuiltinFunction(w.name, w.fn)); err != nil { diff --git a/module/_testinternalcapi/codevar.go b/module/_testinternalcapi/codevar.go new file mode 100644 index 000000000..bba616cf5 --- /dev/null +++ b/module/_testinternalcapi/codevar.go @@ -0,0 +1,588 @@ +// Port of the code-object introspection helpers CPython exposes to the +// test suite through _testinternalcapi: get_code_var_counts, +// get_co_localskinds, code_returns_only_none, and verify_stateless_code. +// The heavy lifting (var-count tallying, unbound-name identification, +// statelessness verification, returns-only-None analysis) mirrors the +// matching functions in Objects/codeobject.c one-for-one. +// +// CPython: Modules/_testinternalcapi.c:969 code_returns_only_none +// CPython: Modules/_testinternalcapi.c:992 get_co_localskinds +// CPython: Modules/_testinternalcapi.c:1023 get_code_var_counts +// CPython: Modules/_testinternalcapi.c:1192 verify_stateless_code +// CPython: Objects/codeobject.c:1818 identify_unbound_names +// CPython: Objects/codeobject.c:1913 _PyCode_GetVarCounts +// CPython: Objects/codeobject.c:2020 _PyCode_SetUnboundVarCounts +// CPython: Objects/codeobject.c:2085 _PyCode_CheckNoInternalState +// CPython: Objects/codeobject.c:2104 _PyCode_CheckNoExternalState +// CPython: Objects/codeobject.c:2132 _PyCode_VerifyStateless +// CPython: Objects/codeobject.c:2166 _PyCode_CheckPureFunction +// CPython: Objects/codeobject.c:2259 code_returns_only_none +package testinternalcapi + +import ( + "fmt" + + "github.com/tamnd/gopy/compile" + "github.com/tamnd/gopy/objects" + "github.com/tamnd/gopy/specialize" +) + +// argsCounts mirrors the args sub-struct of co_locals_counts. +type argsCounts struct { + total int + numposonly int + numposorkw int + numkwonly int + varargs int + varkwargs int +} + +type cellsCounts struct { + total int + numargs int + numothers int +} + +type hiddenCounts struct { + total int + numpure int + numcells int +} + +type localsCounts struct { + total int + args argsCounts + numpure int + cells cellsCounts + hidden hiddenCounts +} + +type globalsCounts struct { + total int + numglobal int + numbuiltin int + numunknown int +} + +type unboundCounts struct { + total int + globals globalsCounts + numattrs int + numunknown int +} + +// varCounts mirrors _PyCode_var_counts_t. +// +// CPython: Include/internal/pycore_code.h:572 _PyCode_var_counts_t +type varCounts struct { + total int + locals localsCounts + numfree int + unbound unboundCounts +} + +// getVarCounts tallies the locals/cells/free vars from co_localspluskinds +// and seeds the unbound counts from co_names. +// +// CPython: Objects/codeobject.c:1913 _PyCode_GetVarCounts +func getVarCounts(co *objects.Code) varCounts { + var locals localsCounts + numfree := 0 + kinds := co.LocalsplusKinds + for i := 0; i < len(kinds); i++ { + kind := kinds[i] + if kind&objects.CoFastFree != 0 { + numfree++ + continue + } + locals.total++ + if kind&objects.CoFastArg != 0 { + locals.args.total++ + switch { + case kind&objects.CoFastArgVar != 0: + if kind&objects.CoFastArgPos != 0 { + locals.args.varargs = 1 + } else { + locals.args.varkwargs = 1 + } + case kind&objects.CoFastArgPos != 0: + if kind&objects.CoFastArgKw != 0 { + locals.args.numposorkw++ + } else { + locals.args.numposonly++ + } + default: + locals.args.numkwonly++ + } + if kind&objects.CoFastCell != 0 { + locals.cells.total++ + locals.cells.numargs++ + } + } else { + if kind&objects.CoFastCell != 0 { + locals.cells.total++ + locals.cells.numothers++ + if kind&objects.CoFastHidden != 0 { + locals.hidden.total++ + locals.hidden.numcells++ + } + } else { + locals.numpure++ + if kind&objects.CoFastHidden != 0 { + locals.hidden.total++ + locals.hidden.numpure++ + } + } + } + } + + numunbound := len(co.Names) + unbound := unboundCounts{ + total: numunbound, + numunknown: numunbound, + } + + return varCounts{ + total: locals.total + numfree + unbound.total, + locals: locals, + numfree: numfree, + unbound: unbound, + } +} + +// nameAt returns the interned name object at index idx in co_names. +func nameAt(co *objects.Code, idx int) objects.Object { + if idx >= 0 && idx < len(co.NameObjs) && co.NameObjs[idx] != nil { + return co.NameObjs[idx] + } + if idx >= 0 && idx < len(co.Names) { + return objects.NewStr(co.Names[idx]) + } + return objects.NewStr("") +} + +// identifyUnboundNames walks the (deoptimized) bytecode and classifies +// each LOAD_GLOBAL / LOAD_ATTR name against the provided namespaces. +// +// CPython: Objects/codeobject.c:1818 identify_unbound_names +func identifyUnboundNames(co *objects.Code, globalnames, attrnames *objects.Set, globalsns, builtinsns *objects.Dict) (unboundCounts, int, error) { + var unbound unboundCounts + numdupes := 0 + code := specialize.DeoptCode(co.Code) + ncodeunits := len(code) / 2 + for i := 0; i < ncodeunits; { + op := compile.Opcode(code[i*2]) + arg := int(code[i*2+1]) + switch op { + case compile.LOAD_ATTR: + name := nameAt(co, arg>>1) + if ok, err := attrnames.Contains(name); err != nil { + return unbound, 0, err + } else if ok { + break + } + unbound.total++ + unbound.numattrs++ + if err := attrnames.Add(name); err != nil { + return unbound, 0, err + } + if ok, err := globalnames.Contains(name); err != nil { + return unbound, 0, err + } else if ok { + numdupes++ + } + case compile.LOAD_GLOBAL: + name := nameAt(co, arg>>1) + if ok, err := globalnames.Contains(name); err != nil { + return unbound, 0, err + } else if ok { + break + } + unbound.total++ + unbound.globals.total++ + switch { + case globalsns != nil && dictContains(globalsns, name): + unbound.globals.numglobal++ + case builtinsns != nil && dictContains(builtinsns, name): + unbound.globals.numbuiltin++ + default: + unbound.globals.numunknown++ + } + if err := globalnames.Add(name); err != nil { + return unbound, 0, err + } + if ok, err := attrnames.Contains(name); err != nil { + return unbound, 0, err + } else if ok { + numdupes++ + } + } + i += 1 + compile.CacheCount(op) + } + return unbound, numdupes, nil +} + +func dictContains(d *objects.Dict, key objects.Object) bool { + ok, err := d.Contains(key) + return err == nil && ok +} + +// setUnboundVarCounts fills in counts.unbound from the bytecode walk, +// reconciling duplicate names that appear as both globals and attrs. +// +// CPython: Objects/codeobject.c:2020 _PyCode_SetUnboundVarCounts +func setUnboundVarCounts(co *objects.Code, counts *varCounts, globalnames, attrnames *objects.Set, globalsns, builtinsns *objects.Dict) error { + if globalnames == nil { + globalnames = objects.NewSet() + } + if attrnames == nil { + attrnames = objects.NewSet() + } + unbound, numdupes, err := identifyUnboundNames(co, globalnames, attrnames, globalsns, builtinsns) + if err != nil { + return err + } + totalunbound := counts.unbound.total + numdupes + unbound.numunknown = totalunbound - unbound.total + unbound.total = totalunbound + counts.unbound = unbound + counts.total += numdupes + return nil +} + +// checkNoExternalState mirrors _PyCode_CheckNoExternalState; returns the +// CPython error message when the code relies on closures or globals. +// +// CPython: Objects/codeobject.c:2104 _PyCode_CheckNoExternalState +func checkNoExternalState(counts *varCounts) string { + switch { + case counts.numfree > 0: + return "closures not supported" + case counts.unbound.globals.numglobal > 0: + return "globals not supported" + case counts.unbound.globals.numbuiltin > 0 && counts.unbound.globals.numunknown > 0: + return "globals not supported" + } + return "" +} + +// checkPureFunction mirrors _PyCode_CheckPureFunction. +// +// CPython: Objects/codeobject.c:2166 _PyCode_CheckPureFunction +func checkPureFunction(co *objects.Code) bool { + flags := uint32(co.Flags) + if flags&compile.CoGenerator != 0 || + flags&compile.CoCoroutine != 0 || + flags&compile.CoIterableCoroutine != 0 || + flags&compile.CoAsyncGenerator != 0 { + return false + } + return true +} + +// returnsOnlyNone mirrors code_returns_only_none: a bare/implicit return +// or an explicit "return None" everywhere (or a function that only raises). +// +// CPython: Objects/codeobject.c:2259 code_returns_only_none +func returnsOnlyNone(co *objects.Code) bool { + if !checkPureFunction(co) { + return false + } + code := specialize.DeoptCode(co.Code) + ncodeunits := len(code) / 2 + if ncodeunits == 0 { + return true + } + + finalOp := compile.Opcode(code[(ncodeunits-1)*2]) + + // Look up None in co_consts. + co.SyncConstObjs() + noneIndex := 0 + nconsts := len(co.ConstObjs) + for ; noneIndex < nconsts; noneIndex++ { + if co.ConstObjs[noneIndex] == objects.None() { + break + } + } + + isReturn := func(op compile.Opcode) bool { return op == compile.RETURN_VALUE } + isRaise := func(op compile.Opcode) bool { + return op == compile.RAISE_VARARGS || op == compile.RERAISE + } + + if noneIndex == nconsts { + // None wasn't in co_consts: no implicit return / "return None". + if isReturn(finalOp) { + return false + } + _ = isRaise // it must end with a raise + for i := 0; i < ncodeunits; { + op := compile.Opcode(code[i*2]) + if isReturn(op) { + return false + } + i += 1 + compile.CacheCount(op) + } + return true + } + + // Walk the bytecode, looking for a RETURN_VALUE that does not return + // the None constant. + for i := 0; i < ncodeunits; { + op := compile.Opcode(code[i*2]) + if isReturn(op) && i != 0 { + prevOp := compile.Opcode(code[(i-1)*2]) + prevArg := int(code[(i-1)*2+1]) + if prevOp == compile.LOAD_CONST && prevArg == noneIndex { + i += 1 + compile.CacheCount(op) + continue + } + return false + } + i += 1 + compile.CacheCount(op) + } + return true +} + +// codeArg resolves the first positional argument to a *Code, unwrapping a +// function and (optionally) sourcing its globals/builtins namespaces. +func codeArg(arg objects.Object, globalsns, builtinsns **objects.Dict) (*objects.Code, error) { + switch v := arg.(type) { + case *objects.Function: + if *globalsns == nil { + if d, ok := v.Globals.(*objects.Dict); ok { + *globalsns = d + } + } + if *builtinsns == nil { + if d, ok := v.Builtins.(*objects.Dict); ok { + *builtinsns = d + } + } + return v.Code, nil + case *objects.Code: + return v, nil + default: + return nil, fmt.Errorf("TypeError: argument must be a code object or a function") + } +} + +// buildVarCountsDict renders a varCounts value into the nested dict the +// test compares against. +func buildVarCountsDict(c *varCounts) (*objects.Dict, error) { + mk := func(pairs [][2]any) (*objects.Dict, error) { + d := objects.NewDict() + for _, p := range pairs { + var val objects.Object + switch v := p[1].(type) { + case int: + val = objects.NewInt(int64(v)) + case *objects.Dict: + val = v + } + if err := d.SetItem(objects.NewStr(p[0].(string)), val); err != nil { + return nil, err + } + } + return d, nil + } + + args, err := mk([][2]any{ + {"total", c.locals.args.total}, + {"numposonly", c.locals.args.numposonly}, + {"numposorkw", c.locals.args.numposorkw}, + {"numkwonly", c.locals.args.numkwonly}, + {"varargs", c.locals.args.varargs}, + {"varkwargs", c.locals.args.varkwargs}, + }) + if err != nil { + return nil, err + } + cells, err := mk([][2]any{ + {"total", c.locals.cells.total}, + {"numargs", c.locals.cells.numargs}, + {"numothers", c.locals.cells.numothers}, + }) + if err != nil { + return nil, err + } + hidden, err := mk([][2]any{ + {"total", c.locals.hidden.total}, + {"numpure", c.locals.hidden.numpure}, + {"numcells", c.locals.hidden.numcells}, + }) + if err != nil { + return nil, err + } + locals, err := mk([][2]any{ + {"total", c.locals.total}, + {"args", args}, + {"numpure", c.locals.numpure}, + {"cells", cells}, + {"hidden", hidden}, + }) + if err != nil { + return nil, err + } + globals, err := mk([][2]any{ + {"total", c.unbound.globals.total}, + {"numglobal", c.unbound.globals.numglobal}, + {"numbuiltin", c.unbound.globals.numbuiltin}, + {"numunknown", c.unbound.globals.numunknown}, + }) + if err != nil { + return nil, err + } + unbound, err := mk([][2]any{ + {"total", c.unbound.total}, + {"globals", globals}, + {"numattrs", c.unbound.numattrs}, + {"numunknown", c.unbound.numunknown}, + }) + if err != nil { + return nil, err + } + return mk([][2]any{ + {"total", c.total}, + {"locals", locals}, + {"numfree", c.numfree}, + {"unbound", unbound}, + }) +} + +// getCodeVarCounts is _testinternalcapi.get_code_var_counts. +// +// CPython: Modules/_testinternalcapi.c:1023 get_code_var_counts +func getCodeVarCounts(args []objects.Object, kwargs map[string]objects.Object) (objects.Object, error) { + if len(args) < 1 { + return nil, fmt.Errorf("TypeError: get_code_var_counts() missing required argument 'code'") + } + var globalnames, attrnames *objects.Set + var globalsns, builtinsns *objects.Dict + if v, ok := kwargs["globalnames"]; ok { + if s, ok := v.(*objects.Set); ok { + globalnames = s + } + } + if v, ok := kwargs["attrnames"]; ok { + if s, ok := v.(*objects.Set); ok { + attrnames = s + } + } + if v, ok := kwargs["globalsns"]; ok { + d, ok := v.(*objects.Dict) + if !ok { + return nil, fmt.Errorf("TypeError: globalsns must be a dict") + } + globalsns = d + } + if v, ok := kwargs["builtinsns"]; ok { + d, ok := v.(*objects.Dict) + if !ok { + return nil, fmt.Errorf("TypeError: builtinsns must be a dict") + } + builtinsns = d + } + + code, err := codeArg(args[0], &globalsns, &builtinsns) + if err != nil { + return nil, err + } + + counts := getVarCounts(code) + if err := setUnboundVarCounts(code, &counts, globalnames, attrnames, globalsns, builtinsns); err != nil { + return nil, err + } + return buildVarCountsDict(&counts) +} + +// getCoLocalskinds is _testinternalcapi.get_co_localskinds. +// +// CPython: Modules/_testinternalcapi.c:992 get_co_localskinds +func getCoLocalskinds(args []objects.Object, kwargs map[string]objects.Object) (objects.Object, error) { + if len(args) != 1 { + return nil, fmt.Errorf("TypeError: get_co_localskinds() takes exactly one argument") + } + co, ok := args[0].(*objects.Code) + if !ok { + return nil, fmt.Errorf("TypeError: argument must be a code object") + } + d := objects.NewDict() + for offset := 0; offset < co.Nlocalsplus && offset < len(co.LocalsplusKinds); offset++ { + name := objects.NewStr(co.LocalsplusNames[offset]) + kind := objects.NewInt(int64(co.LocalsplusKinds[offset])) + if err := d.SetItem(name, kind); err != nil { + return nil, err + } + } + return d, nil +} + +// codeReturnsOnlyNone is _testinternalcapi.code_returns_only_none. +// +// CPython: Modules/_testinternalcapi.c:969 code_returns_only_none +func codeReturnsOnlyNone(args []objects.Object, kwargs map[string]objects.Object) (objects.Object, error) { + if len(args) != 1 { + return nil, fmt.Errorf("TypeError: code_returns_only_none() takes exactly one argument") + } + co, ok := args[0].(*objects.Code) + if !ok { + return nil, fmt.Errorf("TypeError: argument must be a code object") + } + return objects.NewBool(returnsOnlyNone(co)), nil +} + +// verifyStatelessCode is _testinternalcapi.verify_stateless_code. +// +// CPython: Modules/_testinternalcapi.c:1192 verify_stateless_code +// CPython: Objects/codeobject.c:2132 _PyCode_VerifyStateless +func verifyStatelessCode(args []objects.Object, kwargs map[string]objects.Object) (objects.Object, error) { + if len(args) < 1 { + return nil, fmt.Errorf("TypeError: verify_stateless_code() missing required argument 'code'") + } + var globalnames *objects.Set + var globalsns, builtinsns *objects.Dict + if v, ok := kwargs["globalnames"]; ok { + s, ok := v.(*objects.Set) + if !ok { + return nil, fmt.Errorf("TypeError: globalnames must be a set") + } + globalnames = s + } + if v, ok := kwargs["globalsns"]; ok { + d, ok := v.(*objects.Dict) + if !ok { + return nil, fmt.Errorf("TypeError: globalsns must be a dict") + } + globalsns = d + } + if v, ok := kwargs["builtinsns"]; ok { + d, ok := v.(*objects.Dict) + if !ok { + return nil, fmt.Errorf("TypeError: builtinsns must be a dict") + } + builtinsns = d + } + + code, err := codeArg(args[0], &globalsns, &builtinsns) + if err != nil { + return nil, err + } + + counts := getVarCounts(code) + if err := setUnboundVarCounts(code, &counts, globalnames, nil, globalsns, builtinsns); err != nil { + return nil, err + } + // gopy code objects carry no co_extra, so the no-internal-state check + // always passes (CPython: _PyCode_CheckNoInternalState). + if builtinsns != nil { + // Make sure the external-state check fails for globals even when + // there are no builtins. + counts.unbound.globals.numbuiltin++ + } + if msg := checkNoExternalState(&counts); msg != "" { + return nil, fmt.Errorf("ValueError: %s", msg) + } + return objects.None(), nil +} diff --git a/module/_testinternalcapi/instrseq.go b/module/_testinternalcapi/instrseq.go new file mode 100644 index 000000000..47a4da870 --- /dev/null +++ b/module/_testinternalcapi/instrseq.go @@ -0,0 +1,662 @@ +// InstructionSequence object and the compiler-pipeline helpers +// _testinternalcapi exposes to the test suite: new_instruction_sequence, +// compiler_codegen, optimize_cfg, and assemble_code_object. These drive +// gopy's codegen / cfg / assemble passes directly so test_peepholer, +// test_compiler_assemble, test_compiler_codegen, and the vendored +// test.support.bytecode_helper harness run against the real pipeline. +// +// CPython: Python/instruction_sequence.c InstructionSequenceType +// CPython: Modules/_testinternalcapi.c:713 new_instruction_sequence ff. +package testinternalcapi + +import ( + "fmt" + "math/big" + "sort" + + "github.com/tamnd/gopy/ast" + "github.com/tamnd/gopy/builtins" + "github.com/tamnd/gopy/compile" + "github.com/tamnd/gopy/objects" +) + +// instructionSequence wraps a compile.Sequence as the Python-visible +// InstructionSequence object. +// +// CPython: Python/instruction_sequence.c _PyInstructionSequence +type instructionSequence struct { + objects.Header + seq *compile.Sequence +} + +// instructionSequenceType is the Python type for InstructionSequence. +// +// CPython: Python/instruction_sequence.c:460 _PyInstructionSequence_Type +var instructionSequenceType = objects.NewType("InstructionSequence", []*objects.Type{objects.ObjectType()}) + +func init() { + t := instructionSequenceType + methods := []struct { + name string + fn func([]objects.Object, map[string]objects.Object) (objects.Object, error) + }{ + {"addop", instrSeqAddop}, + {"use_label", instrSeqUseLabel}, + {"new_label", instrSeqNewLabel}, + {"add_nested", instrSeqAddNested}, + {"get_nested", instrSeqGetNested}, + {"get_instructions", instrSeqGetInstructions}, + } + for _, m := range methods { + objects.SetTypeDescr(t, m.name, objects.NewMethodDescr(t, m.name, m.fn)) + } +} + +// newInstrSeq wraps seq in a fresh InstructionSequence object. +func newInstrSeq(seq *compile.Sequence) *instructionSequence { + o := &instructionSequence{seq: seq} + o.Init(instructionSequenceType) + return o +} + +// asInt coerces a Python int-like argument to a Go int via __index__. +func asInt(o objects.Object) (int, error) { + idx, err := objects.NumberIndex(o) + if err != nil { + return 0, err + } + i, ok := idx.(*objects.Int) + if !ok { + return 0, fmt.Errorf("TypeError: expected an integer") + } + v, ok := i.Int64() + if !ok { + return 0, fmt.Errorf("OverflowError: Python int too large to convert to C int") + } + return int(v), nil +} + +// instrSeqAddop implements InstructionSequence.addop(opcode, oparg, +// lineno, col_offset, end_lineno, end_col_offset). +// +// CPython: Python/instruction_sequence.c:282 InstructionSequenceType_addop_impl +func instrSeqAddop(args []objects.Object, _ map[string]objects.Object) (objects.Object, error) { + self, ok := args[0].(*instructionSequence) + if !ok { + return nil, fmt.Errorf("TypeError: descriptor 'addop' requires an InstructionSequence") + } + if len(args) != 7 { + return nil, fmt.Errorf("TypeError: addop() takes exactly 6 arguments (%d given)", len(args)-1) + } + vals := make([]int, 6) + for i := 0; i < 6; i++ { + v, err := asInt(args[i+1]) + if err != nil { + return nil, err + } + vals[i] = v + } + opcode, oparg, lineno, colOffset, endLineno, endColOffset := vals[0], vals[1], vals[2], vals[3], vals[4], vals[5] + if err := self.seq.AddopRaw(compile.Opcode(opcode), int32(oparg), lineno, colOffset, endLineno, endColOffset); err != nil { + return nil, fmt.Errorf("ValueError: %s", err.Error()) + } + return objects.None(), nil +} + +// instrSeqUseLabel implements InstructionSequence.use_label(label). +// +// CPython: Python/instruction_sequence.c:257 InstructionSequenceType_use_label_impl +func instrSeqUseLabel(args []objects.Object, _ map[string]objects.Object) (objects.Object, error) { + self, ok := args[0].(*instructionSequence) + if !ok { + return nil, fmt.Errorf("TypeError: descriptor 'use_label' requires an InstructionSequence") + } + if len(args) != 2 { + return nil, fmt.Errorf("TypeError: use_label() takes exactly one argument (%d given)", len(args)-1) + } + label, err := asInt(args[1]) + if err != nil { + return nil, err + } + self.seq.UseLabelID(label) + return objects.None(), nil +} + +// instrSeqNewLabel implements InstructionSequence.new_label() -> int. +// +// CPython: Python/instruction_sequence.c:301 InstructionSequenceType_new_label_impl +func instrSeqNewLabel(args []objects.Object, _ map[string]objects.Object) (objects.Object, error) { + self, ok := args[0].(*instructionSequence) + if !ok { + return nil, fmt.Errorf("TypeError: descriptor 'new_label' requires an InstructionSequence") + } + return objects.NewInt(int64(self.seq.NewLabelID())), nil +} + +// instrSeqAddNested implements InstructionSequence.add_nested(seq). +// +// CPython: Python/instruction_sequence.c:317 InstructionSequenceType_add_nested_impl +func instrSeqAddNested(args []objects.Object, _ map[string]objects.Object) (objects.Object, error) { + self, ok := args[0].(*instructionSequence) + if !ok { + return nil, fmt.Errorf("TypeError: descriptor 'add_nested' requires an InstructionSequence") + } + if len(args) != 2 { + return nil, fmt.Errorf("TypeError: add_nested() takes exactly one argument (%d given)", len(args)-1) + } + nested, ok := args[1].(*instructionSequence) + if !ok { + return nil, fmt.Errorf("TypeError: expected an instruction sequence, not %s", args[1].Type().Name) + } + self.seq.AddNested(nested.seq) + return objects.None(), nil +} + +// instrSeqGetNested implements InstructionSequence.get_nested() -> list. +// Each nested compile.Sequence is wrapped in a fresh InstructionSequence. +// +// CPython: Python/instruction_sequence.c:340 InstructionSequenceType_get_nested_impl +func instrSeqGetNested(args []objects.Object, _ map[string]objects.Object) (objects.Object, error) { + self, ok := args[0].(*instructionSequence) + if !ok { + return nil, fmt.Errorf("TypeError: descriptor 'get_nested' requires an InstructionSequence") + } + items := make([]objects.Object, len(self.seq.Nested)) + for i, child := range self.seq.Nested { + items[i] = newInstrSeq(child) + } + return objects.NewList(items), nil +} + +// instrSeqGetInstructions implements InstructionSequence.get_instructions(). +// Returns one (opcode, oparg|None, lineno, end_lineno, col_offset, +// end_col_offset) tuple per instruction, after resolving jump labels. +// +// CPython: Python/instruction_sequence.c:356 InstructionSequenceType_get_instructions_impl +func instrSeqGetInstructions(args []objects.Object, _ map[string]objects.Object) (objects.Object, error) { + self, ok := args[0].(*instructionSequence) + if !ok { + return nil, fmt.Errorf("TypeError: descriptor 'get_instructions' requires an InstructionSequence") + } + self.seq.ApplyJumpLabels() + items := make([]objects.Object, len(self.seq.Instrs)) + for i := range self.seq.Instrs { + ins := self.seq.Instrs[i] + var arg objects.Object + if ins.Op.HasArg() { + arg = objects.NewInt(int64(ins.Oparg)) + } else { + arg = objects.None() + } + loc := ins.Loc + items[i] = objects.NewTuple([]objects.Object{ + objects.NewInt(int64(ins.Op)), + arg, + objects.NewInt(int64(loc.Lineno)), + objects.NewInt(int64(loc.EndLineno)), + objects.NewInt(int64(loc.ColOffset)), + objects.NewInt(int64(loc.EndColOffset)), + }) + } + return objects.NewList(items), nil +} + +// newInstructionSequence implements _testinternalcapi.new_instruction_sequence(). +// +// CPython: Modules/_testinternalcapi.c:713 new_instruction_sequence +func newInstructionSequence(_ []objects.Object, _ map[string]objects.Object) (objects.Object, error) { + return newInstrSeq(compile.NewSequence()), nil +} + +// compilerCodegen implements compiler_codegen(ast, filename, optimize, +// compile_mode=0) -> (InstructionSequence, metadata). metadata maps +// argcount / posonlyargcount / kwonlyargcount to ints and consts to a +// list in LOAD_CONST index order. +// +// CPython: Modules/_testinternalcapi.c:742 compiler_codegen + +// Python/compile.c:1607 _PyCompile_CodeGen +func compilerCodegen(args []objects.Object, _ map[string]objects.Object) (objects.Object, error) { + if len(args) < 3 { + return nil, fmt.Errorf("TypeError: compiler_codegen() missing required arguments") + } + mod, ok, err := builtins.PyASTObjectToMod(args[0]) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("TypeError: expected an AST") + } + filename, ok := args[1].(*objects.Unicode) + if !ok { + return nil, fmt.Errorf("TypeError: compiler_codegen() filename must be str, not %s", args[1].Type().Name) + } + optimize, err := asInt(args[2]) + if err != nil { + return nil, err + } + unit, err := compile.CodeGenForTest(mod, filename.Value(), optimize) + if err != nil { + return nil, err + } + unit.Seq.ApplyJumpLabels() + + metadata := objects.NewDict() + if err := metadata.SetItem(objects.NewStr("argcount"), objects.NewInt(int64(unit.Argcount))); err != nil { + return nil, err + } + if err := metadata.SetItem(objects.NewStr("posonlyargcount"), objects.NewInt(int64(unit.PosOnlyArgCount))); err != nil { + return nil, err + } + if err := metadata.SetItem(objects.NewStr("kwonlyargcount"), objects.NewInt(int64(unit.KwOnlyArgCount))); err != nil { + return nil, err + } + constItems := make([]objects.Object, len(unit.Consts)) + for i, c := range unit.Consts { + o, err := constToObj(c) + if err != nil { + return nil, err + } + constItems[i] = o + } + if err := metadata.SetItem(objects.NewStr("consts"), objects.NewList(constItems)); err != nil { + return nil, err + } + return objects.NewTuple([]objects.Object{newInstrSeq(unit.Seq), metadata}), nil +} + +// optimizeCfg implements optimize_cfg(instructions, consts, nlocals). +// instructions is an InstructionSequence, consts a list aligned with +// LOAD_CONST opargs. The consts list is mutated in place to reflect the +// optimized const table, and a new optimized InstructionSequence is +// returned. +// +// CPython: Modules/_testinternalcapi.c:767 optimize_cfg + +// Python/flowgraph.c:4126 _PyCompile_OptimizeCfg +func optimizeCfg(args []objects.Object, _ map[string]objects.Object) (objects.Object, error) { + if len(args) != 3 { + return nil, fmt.Errorf("TypeError: optimize_cfg() takes exactly 3 arguments (%d given)", len(args)) + } + seq, ok := args[0].(*instructionSequence) + if !ok { + return nil, fmt.Errorf("ValueError: expected an instruction sequence") + } + constsList, ok := args[1].(*objects.List) + if !ok { + return nil, fmt.Errorf("TypeError: consts must be a list") + } + nlocals, err := asInt(args[2]) + if err != nil { + return nil, err + } + consts := make([]any, constsList.Len()) + for i := 0; i < constsList.Len(); i++ { + c, err := objToConst(constsList.Item(i)) + if err != nil { + return nil, err + } + consts[i] = c + } + out, err := compile.OptimizeCfgForTest(seq.seq, &consts, nlocals) + if err != nil { + return nil, err + } + newConsts := make([]objects.Object, len(consts)) + for i, c := range consts { + o, cerr := constToObj(c) + if cerr != nil { + return nil, cerr + } + newConsts[i] = o + } + constsList.SetSlice(0, constsList.Len(), newConsts) + return newInstrSeq(out), nil +} + +// assembleCodeObject implements assemble_code_object(filename, +// instructions, metadata) -> code. metadata mirrors the C +// _PyCompile_CodeUnitMetadata: name/qualname strings, consts/names/ +// varnames/cellvars/freevars/fasthidden dicts keyed by index, and the +// argcount/posonlyargcount/kwonlyargcount/firstlineno ints. +// +// CPython: Modules/_testinternalcapi.c:795 assemble_code_object + +// Python/compile.c:1657 _PyCompile_Assemble +func assembleCodeObject(args []objects.Object, _ map[string]objects.Object) (objects.Object, error) { + if len(args) != 3 { + return nil, fmt.Errorf("TypeError: assemble_code_object() takes exactly 3 arguments (%d given)", len(args)) + } + filename, ok := args[0].(*objects.Unicode) + if !ok { + return nil, fmt.Errorf("TypeError: assemble_code_object() filename must be str, not %s", args[0].Type().Name) + } + seq, ok := args[1].(*instructionSequence) + if !ok { + return nil, fmt.Errorf("TypeError: expected an instruction sequence") + } + meta, ok := args[2].(*objects.Dict) + if !ok { + return nil, fmt.Errorf("TypeError: metadata must be a dict") + } + unit, err := unitFromMetadata(meta) + if err != nil { + return nil, err + } + co, err := compile.AssembleForTest(unit, seq.seq, filename.Value()) + if err != nil { + return nil, err + } + return builtins.LiftCompileCode(co), nil +} + +// unitFromMetadata builds a compile.Unit from the assemble_code_object +// metadata dict. +// +// CPython: Modules/_testinternalcapi.c:802 assemble_code_object_impl +func unitFromMetadata(meta *objects.Dict) (*compile.Unit, error) { + getStr := func(key string) (string, error) { + v, err := meta.GetItem(objects.NewStr(key)) + if err != nil { + return "", err + } + s, ok := v.(*objects.Unicode) + if !ok { + return "", fmt.Errorf("TypeError: metadata[%q] must be str", key) + } + return s.Value(), nil + } + getInt := func(key string) (int, error) { + v, err := meta.GetItem(objects.NewStr(key)) + if err != nil { + return 0, err + } + return asInt(v) + } + + name, err := getStr("name") + if err != nil { + return nil, err + } + qualname, err := getStr("qualname") + if err != nil { + return nil, err + } + unit := &compile.Unit{Name: name, Qualname: qualname} + + if unit.Argcount, err = getInt("argcount"); err != nil { + return nil, err + } + if unit.PosOnlyArgCount, err = getInt("posonlyargcount"); err != nil { + return nil, err + } + if unit.KwOnlyArgCount, err = getInt("kwonlyargcount"); err != nil { + return nil, err + } + if unit.FirstLineno, err = getInt("firstlineno"); err != nil { + return nil, err + } + + consts, err := constsFromIndexDict(meta, "consts") + if err != nil { + return nil, err + } + unit.Consts = consts + + for _, f := range []struct { + key string + dst *[]string + }{ + {"names", &unit.Names}, + {"varnames", &unit.VarNames}, + {"cellvars", &unit.CellVars}, + {"freevars", &unit.FreeVars}, + } { + names, nerr := namesFromIndexDict(meta, f.key) + if nerr != nil { + return nil, nerr + } + *f.dst = names + } + + fasthidden, err := fasthiddenFromDict(meta) + if err != nil { + return nil, err + } + unit.FastHidden = fasthidden + return unit, nil +} + +// constsFromIndexDict reads a {const: index} dict into a []any ordered +// by index. Mirrors consts_dict_keys_inorder's inverse. +// +// CPython: Python/compile.c consts_dict_keys_inorder +func constsFromIndexDict(meta *objects.Dict, key string) ([]any, error) { + v, err := meta.GetItem(objects.NewStr(key)) + if err != nil { + return nil, err + } + d, ok := v.(*objects.Dict) + if !ok { + return nil, fmt.Errorf("TypeError: metadata[%q] must be a dict", key) + } + type kv struct { + val any + idx int + } + keys := d.Keys() + pairs := make([]kv, len(keys)) + for i, k := range keys { + idxObj, gerr := d.GetItem(k) + if gerr != nil { + return nil, gerr + } + idx, ierr := asInt(idxObj) + if ierr != nil { + return nil, ierr + } + c, cerr := objToConst(k) + if cerr != nil { + return nil, cerr + } + pairs[i] = kv{val: c, idx: idx} + } + sort.Slice(pairs, func(i, j int) bool { return pairs[i].idx < pairs[j].idx }) + out := make([]any, len(pairs)) + for i, p := range pairs { + out[i] = p.val + } + return out, nil +} + +// namesFromIndexDict reads a {name: index} dict into a []string ordered +// by index. +func namesFromIndexDict(meta *objects.Dict, key string) ([]string, error) { + v, err := meta.GetItem(objects.NewStr(key)) + if err != nil { + return nil, err + } + d, ok := v.(*objects.Dict) + if !ok { + return nil, fmt.Errorf("TypeError: metadata[%q] must be a dict", key) + } + type kv struct { + name string + idx int + } + keys := d.Keys() + pairs := make([]kv, len(keys)) + for i, k := range keys { + s, ok := k.(*objects.Unicode) + if !ok { + return nil, fmt.Errorf("TypeError: metadata[%q] keys must be str", key) + } + idxObj, gerr := d.GetItem(k) + if gerr != nil { + return nil, gerr + } + idx, ierr := asInt(idxObj) + if ierr != nil { + return nil, ierr + } + pairs[i] = kv{name: s.Value(), idx: idx} + } + sort.Slice(pairs, func(i, j int) bool { return pairs[i].idx < pairs[j].idx }) + out := make([]string, len(pairs)) + for i, p := range pairs { + out[i] = p.name + } + return out, nil +} + +// fasthiddenFromDict reads the {name: bool/index} fasthidden dict into a +// map[string]bool. +func fasthiddenFromDict(meta *objects.Dict) (map[string]bool, error) { + v, err := meta.GetItem(objects.NewStr("fasthidden")) + if err != nil { + return nil, err + } + d, ok := v.(*objects.Dict) + if !ok { + return nil, fmt.Errorf("TypeError: metadata[\"fasthidden\"] must be a dict") + } + out := make(map[string]bool) + for _, k := range d.Keys() { + s, ok := k.(*objects.Unicode) + if !ok { + return nil, fmt.Errorf("TypeError: metadata[\"fasthidden\"] keys must be str") + } + val, gerr := d.GetItem(k) + if gerr != nil { + return nil, gerr + } + truthy, terr := objects.IsTruthy(val) + if terr != nil { + return nil, terr + } + out[s.Value()] = truthy + } + return out, nil +} + +// objToConst converts a Python constant object into the native const +// value gopy's cfg / assemble passes consume. +func objToConst(o objects.Object) (any, error) { + switch x := o.(type) { + case nil: + return nil, nil + case *objects.Bool: + v, _ := x.Int64() + return v != 0, nil + case *objects.Int: + if v, ok := x.Int64(); ok { + return v, nil + } + return x.BigInt(), nil + case *objects.Float: + return x.Float64(), nil + case *objects.Complex: + return x.Complex128(), nil + case *objects.Unicode: + return x.Value(), nil + case *objects.Bytes: + return x.Bytes(), nil + case *objects.Tuple: + vals := make([]any, x.Len()) + for i := 0; i < x.Len(); i++ { + c, err := objToConst(x.Item(i)) + if err != nil { + return nil, err + } + vals[i] = c + } + return &compile.ConstTuple{Values: vals}, nil + case *objects.Set: + if o.Type() == objects.FrozensetType { + items := x.Items() + vals := make(ast.FrozenSet, len(items)) + for i, it := range items { + c, err := objToConst(it) + if err != nil { + return nil, err + } + vals[i] = c + } + return vals, nil + } + case *objects.Code: + // A nested code object const round-trips through the assembler + // unchanged: liftCompileConst passes any non-compile value back + // out, so co_consts keeps the same code object. + return x, nil + } + if objects.IsNone(o) { + return nil, nil + } + if o == objects.Ellipsis() { + return ast.EllipsisType{}, nil + } + return nil, fmt.Errorf("ValueError: unsupported constant of type %s", o.Type().Name) +} + +// constToObj is the inverse of objToConst: it turns a native const value +// back into the Python object the test harness inspects. +func constToObj(c any) (objects.Object, error) { + switch x := c.(type) { + case nil: + return objects.None(), nil + case bool: + return objects.NewBool(x), nil + case int: + return objects.NewInt(int64(x)), nil + case int32: + return objects.NewInt(int64(x)), nil + case int64: + return objects.NewInt(x), nil + case *big.Int: + return objects.NewIntFromBig(x), nil + case float64: + return objects.NewFloat(x), nil + case complex128: + return objects.NewComplex(real(x), imag(x)), nil + case string: + return objects.NewStr(x), nil + case []byte: + return objects.NewBytes(x), nil + case *compile.ConstTuple: + items := make([]objects.Object, len(x.Values)) + for i, v := range x.Values { + o, err := constToObj(v) + if err != nil { + return nil, err + } + items[i] = o + } + return objects.NewTuple(items), nil + case []any: + items := make([]objects.Object, len(x)) + for i, v := range x { + o, err := constToObj(v) + if err != nil { + return nil, err + } + items[i] = o + } + return objects.NewTuple(items), nil + case ast.FrozenSet: + items := make([]objects.Object, len(x)) + for i, v := range x { + o, err := constToObj(v) + if err != nil { + return nil, err + } + items[i] = o + } + return objects.NewFrozenset(items) + case ast.EllipsisType: + return objects.Ellipsis(), nil + case *compile.Code: + return builtins.LiftCompileCode(x), nil + case objects.Object: + return x, nil + default: + return nil, fmt.Errorf("ValueError: unsupported constant of type %T", c) + } +} diff --git a/module/_testinternalcapi/module.go b/module/_testinternalcapi/module.go index c1fd96a04..eff4399bf 100644 --- a/module/_testinternalcapi/module.go +++ b/module/_testinternalcapi/module.go @@ -37,6 +37,14 @@ func buildModule() (*objects.Module, error) { {"run_in_subinterp_with_config", runInSubinterpWithConfig}, {"clear_extension", clearExtension}, {"dict_getitem_knownhash", dictGetitemKnownhash}, + {"get_code_var_counts", getCodeVarCounts}, + {"get_co_localskinds", getCoLocalskinds}, + {"code_returns_only_none", codeReturnsOnlyNone}, + {"verify_stateless_code", verifyStatelessCode}, + {"new_instruction_sequence", newInstructionSequence}, + {"compiler_codegen", compilerCodegen}, + {"optimize_cfg", optimizeCfg}, + {"assemble_code_object", assembleCodeObject}, } for _, f := range fns { if err := d.SetItem(objects.NewStr(f.name), objects.NewBuiltinFunction(f.name, f.fn)); err != nil { diff --git a/module/sys/helpers.go b/module/sys/helpers.go index aedc0dae9..139ccd559 100644 --- a/module/sys/helpers.go +++ b/module/sys/helpers.go @@ -355,10 +355,10 @@ func isTypeError(err error) bool { return strings.HasPrefix(err.Error(), "TypeError:") } -// makeIntern ports sys.intern. The interned table lands with the -// unicodeobject port (1616); until then gopy returns the input string -// unchanged so callers see the round-trip semantics. Non-str input -// raises TypeError per CPython. +// makeIntern ports sys.intern. It runs the argument through the global +// intern table (PyUnicode_InternInPlace) and hands back the canonical +// pointer, so two equal strings that are both interned compare identical. +// Non-str input raises TypeError per CPython. // // CPython: Python/sysmodule.c:1004 sys_intern_impl func makeIntern(ts *state.Thread) func([]objects.Object, map[string]objects.Object) (objects.Object, error) { @@ -371,6 +371,8 @@ func makeIntern(ts *state.Thread) func([]objects.Object, map[string]objects.Obje errors.SetString(ts, errors.PyExc_TypeError, msg) return nil, fmt.Errorf("TypeError: %s", msg) } - return args[0], nil + s := args[0] + objects.InternInPlace(&s) + return s, nil } } diff --git a/module/sys/module.go b/module/sys/module.go index b2dcd4832..5d5c44f53 100644 --- a/module/sys/module.go +++ b/module/sys/module.go @@ -116,6 +116,29 @@ func SetStdlibDir(dir string) { // CPython: Python/initconfig.c:1828 config_init_safe_path var pendingSafePath bool +// pendingOptimize records the optimization level supplied on the command +// line (-O / -OO / PYTHONOPTIMIZE) before sys is built. buildModule reads +// it through pendingConfig when stamping sys.flags. +// +// CPython: Python/initconfig.c:1700 config_init_optimization_level +var pendingOptimize int + +// pendingConfig builds a default PyConfig carrying the command-line knobs +// captured before sys was imported (safe_path, optimization level), so +// every sys.flags rebuild reflects all of them rather than clobbering one +// when another is set. +// +// CPython: Python/sysmodule.c:3478 set_flags_from_config +func pendingConfig() *initconfig.PyConfig { + cfg := &initconfig.PyConfig{} + cfg.InitPythonConfig() + if pendingSafePath { + cfg.SafePath = 1 + } + cfg.OptimizationLevel = pendingOptimize + return cfg +} + // SetSafePath records safe_path and, when sys is already live, rebuilds // sys.flags so sys.flags.safe_path reads True. // @@ -123,12 +146,18 @@ var pendingSafePath bool func SetSafePath(on bool) { pendingSafePath = on if md := liveSysDict(); md != nil { - cfg := &initconfig.PyConfig{} - cfg.InitPythonConfig() - if on { - cfg.SafePath = 1 - } - _ = md.SetItem(objects.NewStr("flags"), makeFlags(cfg)) + _ = md.SetItem(objects.NewStr("flags"), makeFlags(pendingConfig())) + } +} + +// SetOptimize records the optimization level and, when sys is already +// live, rebuilds sys.flags so sys.flags.optimize reads the new level. +// +// CPython: Python/sysmodule.c:3478 set_flags_from_config (optimize) +func SetOptimize(level int) { + pendingOptimize = level + if md := liveSysDict(); md != nil { + _ = md.SetItem(objects.NewStr("flags"), makeFlags(pendingConfig())) } } @@ -723,7 +752,9 @@ func internShim(args []objects.Object, _ map[string]objects.Object) (objects.Obj if args[0].Type() != objects.StrType() { return nil, errInternType(args[0].Type().Name) } - return args[0], nil + s := args[0] + objects.InternInPlace(&s) + return s, nil } func errInternArity(n int) error { diff --git a/objects/bytes_format.go b/objects/bytes_format.go index 60b8cebd1..3f3871645 100644 --- a/objects/bytes_format.go +++ b/objects/bytes_format.go @@ -12,6 +12,7 @@ package objects import ( "bytes" "fmt" + "math" "strings" ) @@ -137,7 +138,7 @@ func bytesFormat(format []byte, args Object, useByteArray bool) (Object, error) pos++ } bytesWidth: - if err := bytesReadNumOrStar(format, &pos, &arg.width, getnextarg); err != nil { + if err := bytesReadNumOrStar(format, &pos, &arg.width, false, getnextarg); err != nil { return nil, err } if arg.width < 0 && arg.width != -1 { @@ -149,7 +150,7 @@ func bytesFormat(format []byte, args Object, useByteArray bool) (Object, error) if pos < len(format) && format[pos] == '.' { pos++ arg.prec = 0 - if err := bytesReadNumOrStar(format, &pos, &arg.prec, getnextarg); err != nil { + if err := bytesReadNumOrStar(format, &pos, &arg.prec, true, getnextarg); err != nil { return nil, err } if arg.prec < 0 { @@ -398,7 +399,17 @@ func bytesWritePadded(out []byte, arg *fmtArg, body []byte) ([]byte, error) { // bytesReadNumOrStar parses a width/precision token at *pos. "*" pulls the // next argument; a digit run reads as decimal. -func bytesReadNumOrStar(format []byte, pos *int, dst *int, getnextarg func() (Object, error)) error { +// +// asInt selects the C type the "*" argument is coerced into: width goes +// through PyLong_AsSsize_t and precision through PyLong_AsInt, so a +// precision above INT_MAX overflows even though it fits a ssize_t. A +// literal digit run is bounded by PY_SSIZE_T_MAX for width and INT_MAX +// for precision, matching the two overflow checks below. +// +// CPython: Objects/bytesobject.c:746 (width PyLong_AsSsize_t), +// +// Objects/bytesobject.c:787 (prec PyLong_AsInt) +func bytesReadNumOrStar(format []byte, pos *int, dst *int, asInt bool, getnextarg func() (Object, error)) error { if *pos >= len(format) { return nil } @@ -414,7 +425,10 @@ func bytesReadNumOrStar(format []byte, pos *int, dst *int, getnextarg func() (Ob } n, fits := i.Int64() if !fits { - return fmt.Errorf("OverflowError: width too big") + return fmt.Errorf("OverflowError: Python int too large to convert to C ssize_t") + } + if asInt && (n > math.MaxInt32 || n < math.MinInt32) { + return fmt.Errorf("OverflowError: Python int too large to convert to C int") } *dst = int(n) return nil @@ -423,19 +437,25 @@ func bytesReadNumOrStar(format []byte, pos *int, dst *int, getnextarg func() (Ob if c < '0' || c > '9' { return nil } - n := 0 + limit := int64(math.MaxInt64) + overMsg := "ValueError: width too big" + if asInt { + limit = math.MaxInt32 + overMsg = "ValueError: prec too big" + } + var n int64 for *pos < len(format) { c := format[*pos] if c < '0' || c > '9' { break } - if n > (1<<31-1-int(c-'0'))/10 { - return fmt.Errorf("ValueError: width too big") + if n > (limit-int64(c-'0'))/10 { + return fmt.Errorf("%s", overMsg) } - n = n*10 + int(c-'0') + n = n*10 + int64(c-'0') *pos++ } - *dst = n + *dst = int(n) return nil } diff --git a/objects/code.go b/objects/code.go index 2cf7412f1..225b3c74a 100644 --- a/objects/code.go +++ b/objects/code.go @@ -183,6 +183,24 @@ type Code struct { // CPython: Include/cpython/code.h:90 co_version Version uint32 + // filenameObj / linetableObj / exceptiontableObj / constsObj are + // the interned Python-object forms of Filename / Linetable / + // ExceptionTable / Consts, populated by InternConstants at the + // lift boundary so co_filename, co_linetable, co_exceptiontable + // and co_consts return a stable, shared object. CPython holds each + // of these as a single PyObject merged through the per-compile + // const_cache (so two functions with identical linetables or + // consts share one object, and a nested code object's co_filename + // is its parent's). When nil, the getset falls back to building a + // fresh object from the Go field, which covers hand-built and + // marshal-loaded code that never passed through the interner. + // + // CPython: Python/compile.c:318 const_cache_insert + filenameObj Object + linetableObj Object + exceptiontableObj Object + constsObj *Tuple + // CacheObjects is gopy's stand-in for CPython's in-cache pointer // slots. CPython packs the cached descriptor / function object // pointer into 4 codeunits of the inline cache (write_obj + @@ -413,6 +431,10 @@ func codeGetAttr(o Object, name Object) (Object, error) { } switch n.v { case "co_filename": + if c.filenameObj != nil { + Incref(c.filenameObj) + return c.filenameObj, nil + } return NewStr(c.Filename), nil case "co_name": return NewStr(c.Name), nil @@ -709,13 +731,41 @@ func constsEqual(a, b []any) bool { return false } for i := range a { - if !reflect.DeepEqual(a[i], b[i]) { + if !constElemEqual(a[i], b[i]) { return false } } return true } +// constElemEqual compares two co_consts entries the way CPython's +// _PyCode_ConstantKey + PyObject_RichCompareBool does: nested code +// objects fall through to code_richcompare (so co_filename and other +// fields it ignores do not break equality), and tuples recurse so a +// code object buried inside a const tuple is handled the same way. +// Everything else uses reflect.DeepEqual on the Go representation. +// +// CPython: Objects/codeobject.c:2634 _PyCode_ConstantKey comparison +func constElemEqual(a, b any) bool { + if ca, ok := a.(*Code); ok { + cb, ok := b.(*Code) + return ok && codeEqual(ca, cb) + } + if ta, ok := a.(*Tuple); ok { + tb, ok := b.(*Tuple) + if !ok || ta.Len() != tb.Len() { + return false + } + for i := 0; i < ta.Len(); i++ { + if !constElemEqual(ta.Item(i), tb.Item(i)) { + return false + } + } + return true + } + return reflect.DeepEqual(a, b) +} + func stringsHash(ss []string) int64 { const mult uint64 = 0xf4243 uhash := uint64(len(ss)) @@ -837,6 +887,13 @@ func (c *Code) Replace(r CodeReplace) (*Code, error) { if r.SetCellvars { out.Cellvars = cloneStrings(r.Cellvars) } + if r.SetVarnames || r.SetCellvars || r.SetFreevars { + // One of the name tables changed, so the flat localsplus layout + // Copy carried over is stale. Rebuild it the way the constructor + // would. CPython: Objects/codeobject.c:2932 routes replace through + // PyCode_NewWithPosOnlyArgs, which recomputes co_localsplusnames. + out.rebuildLocalsplus() + } if r.Filename != nil { out.Filename = *r.Filename } @@ -879,6 +936,20 @@ func (c *Code) Copy() *Code { out.Varnames = cloneStrings(c.Varnames) out.Freevars = cloneStrings(c.Freevars) out.Cellvars = cloneStrings(c.Cellvars) + // The flat localsplus layout (and its derived counts) is part of the + // code object's identity: co_localsplusnames is what _varname_from_oparg, + // the frame allocator, and dis all index. CPython rebuilds it in the + // constructor; carrying it here keeps an unmodified Copy / no-arg + // replace() truly identical instead of dropping the layout and tripping + // "_varname_from_oparg(): oparg out of range". + // + // CPython: Objects/codeobject.c:536 _PyCode_New (co_localsplusnames) + out.LocalsplusNames = cloneStrings(c.LocalsplusNames) + out.LocalsplusKinds = cloneBytes(c.LocalsplusKinds) + out.Nlocalsplus = c.Nlocalsplus + out.Nlocals = c.Nlocals + out.Ncellvars = c.Ncellvars + out.Nfreevars = c.Nfreevars out.Linetable = cloneBytes(c.Linetable) out.ExceptionTable = cloneBytes(c.ExceptionTable) out.SyncNameObjs() @@ -886,6 +957,48 @@ func (c *Code) Copy() *Code { return out } +// rebuildLocalsplus recomputes LocalsplusNames / LocalsplusKinds and the +// derived counts from Varnames / Cellvars / Freevars, mirroring the flat +// layout the constructor builds: varnames first (CO_FAST_LOCAL), then +// cellvars (CO_FAST_CELL, merged into the matching arg slot when a cell +// shares a name with a varname), then freevars (CO_FAST_FREE). Replace +// calls this whenever one of those three name tables changes so the +// localsplus view stays consistent, exactly as code_replace_impl does by +// routing through PyCode_NewWithPosOnlyArgs. +// +// CPython: Objects/codeobject.c:802 _PyCode_New localsplus build +func (c *Code) rebuildLocalsplus() { + names := make([]string, 0, len(c.Varnames)+len(c.Cellvars)+len(c.Freevars)) + kinds := make([]byte, 0, cap(names)) + for _, name := range c.Varnames { + names = append(names, name) + kinds = append(kinds, CoFastLocal) + } + for _, cell := range c.Cellvars { + argoffset := -1 + for j, v := range c.Varnames { + if v == cell { + argoffset = j + break + } + } + if argoffset >= 0 { + // Cell shares a slot with the argument of the same name. + kinds[argoffset] |= CoFastCell + continue + } + names = append(names, cell) + kinds = append(kinds, CoFastCell) + } + for _, free := range c.Freevars { + names = append(names, free) + kinds = append(kinds, CoFastFree) + } + c.LocalsplusNames = names + c.LocalsplusKinds = kinds + c.SyncLocalsplusCounts() +} + func nonNegative(p *int, name string) error { if p != nil && *p < 0 { return fmt.Errorf("ValueError: %s must be a positive integer", name) diff --git a/objects/code_attrs.go b/objects/code_attrs.go index 1fefe3559..b0ad477c9 100644 --- a/objects/code_attrs.go +++ b/objects/code_attrs.go @@ -32,6 +32,17 @@ var DeprecWarnHook func(msg string) error // CPython: Objects/typeobject.c:4667 type_new (PyErr_WarnFormat RuntimeWarning) var RuntimeWarnHook func(msg string) error +// CodeDeoptHook is set by the specialize package at init time so the +// co_code getter can return the deoptimized form of the executable +// bytecode without importing specialize (which imports objects). The +// hook rewrites every specialized opcode back to its adaptive parent +// and zeroes the inline cache cells, returning a fresh slice. nil when +// specialization is not wired (tests / minimal builds), in which case +// co_code returns the raw bytes unchanged. +// +// CPython: Objects/codeobject.c:2310 _PyCode_GetCode (runs deopt_code) +var CodeDeoptHook func(code []byte) []byte + func init() { SetTypeDescr(CodeType, "co_lines", NewMethodDescr(CodeType, "co_lines", codeCoLinesMethod)) SetTypeDescr(CodeType, "co_positions", NewMethodDescr(CodeType, "co_positions", codeCoPositionsMethod)) @@ -45,11 +56,31 @@ func init() { func codeAttrLookup(c *Code, name string) (Object, bool) { switch name { case "co_code": + // co_code is the deoptimized snapshot: specialized opcodes the + // adaptive interpreter wrote in place are mapped back to their + // adaptive parent and the inline cache cells zeroed, so dis and + // marshal observe a stable, specialization-independent form. + // + // CPython: Objects/codeobject.c:2310 _PyCode_GetCode + if CodeDeoptHook != nil { + return NewBytes(CodeDeoptHook(c.Code)), true + } return NewBytes(c.Code), true case "_co_code_adaptive": // CPython: Objects/codeobject.c:2777 code_getcodeadaptive return NewBytes(c.Code), true case "co_consts": + // The cached tuple is a counted reference owned by the code + // object; hand the reader a fresh reference so a transient read + // (e.g. LOAD_ATTR followed by a DECREF) does not draw the pinned + // object down to zero and clear it. CPython's getter returns + // Py_NewRef(co->co_consts). + // + // CPython: Objects/codeobject.c:2724 code_memberlist (T_OBJECT) + if c.constsObj != nil { + Incref(c.constsObj) + return c.constsObj, true + } return constsAsTuple(c.Consts), true case "co_names": return stringsAsTuple(c.Names), true @@ -84,6 +115,10 @@ func codeAttrLookup(c *Code, name string) (Object, bool) { // CPython: Include/cpython/code.h:92 co_localsplusnames return stringsAsTuple(c.LocalsplusNames), true case "co_linetable": + if c.linetableObj != nil { + Incref(c.linetableObj) + return c.linetableObj, true + } return NewBytes(c.Linetable), true case "co_lnotab": // Deprecated since Python 3.12; emit DeprecationWarning. @@ -93,6 +128,10 @@ func codeAttrLookup(c *Code, name string) (Object, bool) { } return NewBytes(c.Linetable), true case "co_exceptiontable": + if c.exceptiontableObj != nil { + Incref(c.exceptiontableObj) + return c.exceptiontableObj, true + } return NewBytes(c.ExceptionTable), true } return nil, false @@ -119,6 +158,42 @@ func stringsAsTuple(ss []string) *Tuple { return NewTuple(items) } +// WrapConstStr lifts a string constant into a *Unicode, interning it +// when it qualifies. CPython interns identifier-like string constants +// when the code object is built (intern_constants, recursing into +// tuples and frozensets); gopy materializes consts lazily, so every +// path that turns a const string into an Object routes through here so +// the interning is uniform whether the const is read via co_consts or +// loaded by the VM. +// +// CPython: Objects/codeobject.c:203 intern_constants +func WrapConstStr(s string) Object { + if shouldInternString(s) { + return InternFromString(s) + } + return NewStr(s) +} + +// shouldInternString reports whether a string constant qualifies for +// interning: ASCII and made up only of [a-zA-Z0-9_]. This is the +// default-build (non free-threaded) predicate; the free-threaded build +// interns every string constant. +// +// CPython: Objects/codeobject.c:117 should_intern_string +func shouldInternString(s string) bool { + for i := 0; i < len(s); i++ { + c := s[i] + if c >= 0x80 { + return false + } + isalnum := (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + if !isalnum && c != '_' { + return false + } + } + return true +} + // wrapConstAttr lifts one raw constant into an Object. Mirrors // vm.wrapConst for the scalar types marshal recovers, but lives in // objects/ so codeGetAttr can use it without an import cycle. Compile @@ -143,7 +218,7 @@ func wrapConstAttr(v any) Object { case complex128: return NewComplex(real(x), imag(x)) case string: - return NewStr(x) + return WrapConstStr(x) case []byte: return NewBytes(x) case *Code: @@ -258,19 +333,15 @@ func codeVarnameFromOpargMethod(args []Object, kwargs map[string]Object) (Object } i64, _ := idxObj.Int64() idx := int(i64) - if idx < 0 { + // The oparg is a localsplus offset, so it indexes co_localsplusnames + // directly. Reconstructing from varnames+cellvars+freevars would + // double-count arg-cells (an argument that is also a cell shares one + // localsplus slot), shifting freevars by the number of such cells. + // + // CPython: Objects/codeobject.c:2955 code__varname_from_oparg_impl + // (PyTuple_GET_ITEM(co->co_localsplusnames, oparg)) + if idx < 0 || idx >= len(c.LocalsplusNames) { return nil, fmt.Errorf("IndexError: _varname_from_oparg(): oparg out of range") } - if idx < len(c.Varnames) { - return NewStr(c.Varnames[idx]), nil - } - idx -= len(c.Varnames) - if idx < len(c.Cellvars) { - return NewStr(c.Cellvars[idx]), nil - } - idx -= len(c.Cellvars) - if idx < len(c.Freevars) { - return NewStr(c.Freevars[idx]), nil - } - return nil, fmt.Errorf("IndexError: _varname_from_oparg(): oparg out of range") + return NewStr(c.LocalsplusNames[idx]), nil } diff --git a/objects/code_attrs_test.go b/objects/code_attrs_test.go index 41f8d1c58..40800ca0c 100644 --- a/objects/code_attrs_test.go +++ b/objects/code_attrs_test.go @@ -74,6 +74,9 @@ func TestCodeVarnameFromOparg(t *testing.T) { c.Varnames = []string{"x", "y"} c.Cellvars = []string{"c"} c.Freevars = []string{"f"} + // _varname_from_oparg indexes co_localsplusnames directly, mirroring + // CPython's code__varname_from_oparg_impl (Objects/codeobject.c:2955). + c.LocalsplusNames = []string{"x", "y", "c", "f"} check := func(arg int64, want string) { t.Helper() diff --git a/objects/code_constcache.go b/objects/code_constcache.go new file mode 100644 index 000000000..613d7c62f --- /dev/null +++ b/objects/code_constcache.go @@ -0,0 +1,191 @@ +package objects + +import ( + "math" + "reflect" + "sort" + "strconv" + "strings" +) + +// constCache merges equal constants across every code object produced +// by a single compile() call. CPython keeps a per-compile dict +// (compiler.c_const_cache) and routes every constant, plus each code +// object's co_consts / co_names / co_linetable / co_exceptiontable / +// co_code through it, so two functions with identical constants or +// identical location tables share one object. gopy materializes +// constants late (the lift from compile.Code to objects.Code), so the +// merge runs there instead: InternCodeConstants walks the freshly +// lifted code tree once and interns each piece through a shared cache. +// +// CPython: Python/compile.c:98 c_const_cache +// CPython: Python/compile.c:318 const_cache_insert +type constCache struct { + byKey map[string]Object +} + +// InternCodeConstants merges the constant / location-table objects of a +// freshly lifted code tree so co_consts, co_linetable, co_exceptiontable +// and co_filename return stable, shared objects. The whole tree compiled +// in one compile() call shares one cache, matching CPython's per-compile +// const_cache; callers invoke this once on the top-level code object. +// +// CPython: Python/compile.c:1707 (const_cache created per compile, then +// every makecode merges its tables through it). +func InternCodeConstants(top *Code) { + if top == nil { + return + } + cc := &constCache{byKey: make(map[string]Object)} + cc.internCode(top, nil) +} + +// internCode interns one code object's tables in place. filenameObj is +// the already-interned co_filename of the enclosing code, threaded down +// so every nested code object shares one filename object exactly as +// CPython passes a single filename PyObject to each makecode call. +func (cc *constCache) internCode(c *Code, filenameObj Object) { + if filenameObj == nil { + filenameObj = cc.intern(NewStr(c.Filename)) + } + // Pin each cached table on the code object. co_filename / co_linetable + // / co_exceptiontable / co_consts are un-counted Go fields, so a merged + // object shared across two code units would otherwise sit at a refcount + // that the first transient Decref drops to zero, tearing the storage + // down while a later co_consts read still points at it. CPython holds + // each of these as a counted reference for the code's lifetime; the + // Incref here is that reference (mirrors SyncConstObjs pinning the + // per-code consts tuple). + c.filenameObj = filenameObj + Incref(filenameObj) + c.linetableObj = cc.intern(NewBytes(c.Linetable)) + Incref(c.linetableObj) + c.exceptiontableObj = cc.intern(NewBytes(c.ExceptionTable)) + Incref(c.exceptiontableObj) + + items := make([]Object, len(c.Consts)) + for i, v := range c.Consts { + obj := wrapConstAttr(v) + if nested, ok := obj.(*Code); ok { + cc.internCode(nested, filenameObj) + } + items[i] = cc.intern(obj) + } + if t, ok := cc.intern(NewTuple(items)).(*Tuple); ok { + c.constsObj = t + Incref(t) + } +} + +// intern returns the canonical object for o, merging equal constants. +// Recurses into tuples and frozensets so their elements are interned +// too, mirroring const_cache_insert's recursive=true mode. +// +// CPython: Python/compile.c:318 const_cache_insert +func (cc *constCache) intern(o Object) Object { + // None and Ellipsis are singletons; no key needed. + if IsNone(o) { + return o + } + if _, ok := o.(*ellipsisObject); ok { + return o + } + switch x := o.(type) { + case *Tuple: + items := make([]Object, x.Len()) + for i := 0; i < x.Len(); i++ { + items[i] = cc.intern(x.Item(i)) + } + merged := NewTuple(items) + key := constKey(merged) + if v, ok := cc.byKey[key]; ok { + return v + } + cc.byKey[key] = merged + return merged + case *Set: + if x.frozen { + items := x.Items() + interned := make([]Object, len(items)) + for i, it := range items { + interned[i] = cc.intern(it) + } + key := frozensetKey(interned) + if v, ok := cc.byKey[key]; ok { + return v + } + // Rebuild the frozenset so its elements are the canonical + // interned objects: the test relies on a tuple stored both as + // a standalone const and as a frozenset member being one + // object. const_cache_insert merges recursively, so the + // member must point at the same interned tuple. + merged, err := NewFrozenset(interned) + if err != nil { + cc.byKey[key] = o + return o + } + cc.byKey[key] = merged + return merged + } + } + key := constKey(o) + if v, ok := cc.byKey[key]; ok { + return v + } + cc.byKey[key] = o + return o +} + +// constKey builds a structural key that distinguishes constants the way +// CPython's _PyCode_ConstantKey does: by exact type and value, so +// True / 1 / 1.0 never collapse, -0.0 differs from 0.0, and NaN keys on +// its bit pattern. Containers recurse on element keys. +// +// CPython: Objects/codeobject.c:2634 _PyCode_ConstantKey +func constKey(o Object) string { + switch x := o.(type) { + case *ellipsisObject: + return "..." + case *Bool: + if x.Sign() != 0 { + return "bool:1" + } + return "bool:0" + case *Int: + return "int:" + x.BigInt().String() + case *Float: + return "float:" + strconv.FormatUint(math.Float64bits(x.Float64()), 16) + case *Complex: + return "complex:" + strconv.FormatUint(math.Float64bits(x.Real()), 16) + + ":" + strconv.FormatUint(math.Float64bits(x.Imag()), 16) + case *Unicode: + return "str:" + x.Value() + case *Bytes: + return "bytes:" + string(x.Bytes()) + case *Tuple: + parts := make([]string, x.Len()) + for i := 0; i < x.Len(); i++ { + parts[i] = constKey(x.Item(i)) + } + return "tuple(" + strings.Join(parts, ",") + ")" + case *Set: + if x.frozen { + return frozensetKey(x.Items()) + } + } + if IsNone(o) { + return "None" + } + // Code objects and anything else are never merged: key on identity. + return "id:" + strconv.FormatUint(uint64(reflect.ValueOf(o).Pointer()), 16) +} + +// frozensetKey is order-independent so {0, 1} and {1, 0} collapse. +func frozensetKey(items []Object) string { + parts := make([]string, len(items)) + for i, it := range items { + parts[i] = constKey(it) + } + sort.Strings(parts) + return "frozenset{" + strings.Join(parts, ",") + "}" +} diff --git a/objects/complex.go b/objects/complex.go index 3f7f3dc95..0c3789051 100644 --- a/objects/complex.go +++ b/objects/complex.go @@ -623,7 +623,7 @@ func complexFormat(o Object, spec string) (string, error) { } s, err := format.ParseSpec(spec) if err != nil { - return "", fmt.Errorf("ValueError: invalid format spec for complex") + return "", formatSpecError(err, spec, o.Type().Name) } if s.Fill == '0' { return "", fmt.Errorf("ValueError: Zero padding is not allowed in complex format specifier") @@ -638,6 +638,11 @@ func complexFormat(o Object, spec string) (string, error) { return "", fmt.Errorf("ValueError: Unknown format code '%c' for object of type 'complex'", origType) } + // CPython: Python/formatter_unicode.c:1338 precision > INT_MAX + if s.Precision > math.MaxInt32 { + return "", fmt.Errorf("ValueError: %w", format.ErrPrecisionTooBig) + } + v := o.(*Complex).v re, im := real(v), imag(v) diff --git a/objects/float_format.go b/objects/float_format.go index 597596c66..d07db3dec 100644 --- a/objects/float_format.go +++ b/objects/float_format.go @@ -75,7 +75,7 @@ func floatFormat(o Object, spec string) (string, error) { } parsed, err := format.ParseSpec(spec) if err != nil { - return "", fmt.Errorf("ValueError: %w", err) + return "", formatSpecError(err, spec, o.Type().Name) } switch parsed.Type { case 0, 'e', 'E', 'f', 'F', 'g', 'G', '%', 'n': diff --git a/objects/frame.go b/objects/frame.go index 3d5a0229c..05549117e 100644 --- a/objects/frame.go +++ b/objects/frame.go @@ -472,6 +472,16 @@ func frameSetAttr(o Object, name Object, v Object) error { return h(f, f.traceOpcodes) } return nil + case "f_lineno": + // Settable f_lineno is the debugger line-jump feature: a trace + // function assigns it to relocate the instruction pointer to a + // chosen source line. The vm installs the hook; without one + // (e.g. objects-only tests) the slot stays read-only. + // + // CPython: Objects/frameobject.c:1640 frame_lineno_set_impl + if h := SetLinenoHook; h != nil { + return h(f, v) + } } return fmt.Errorf("AttributeError: 'frame' object attribute %q is read-only", n.v) } diff --git a/objects/instance.go b/objects/instance.go index 0433d078a..449ca0e6a 100644 --- a/objects/instance.go +++ b/objects/instance.go @@ -96,6 +96,17 @@ var WarnUnawaitedCoroutineHook func(coro Object) // CPython: Python/ceval.c:_PyEval_SetOpcodeTrace var SetOpcodeTraceHook func(f *Frame, enable bool) error +// SetLinenoHook implements frame.f_lineno's setter (the debugger line +// jump). It validates the requested line against the running frame's +// code, models the evaluation stack to pick a compatible bytecode +// offset, binds any newly-live locals to None, pops excess stack +// values, and moves the instruction pointer. objects/ stays free of +// the vm/frame/monitor machinery the implementation needs by routing +// through this hook, installed by the vm at startup. +// +// CPython: Objects/frameobject.c:1640 frame_lineno_set_impl +var SetLinenoHook func(f *Frame, value Object) error + // WarnUnawaitedAgenMethodHook routes a never-awaited async-generator // asend/athrow/aclose awaitable through warnings so the consumer sees a // RuntimeWarning of the form "coroutine method 'asend' of '' diff --git a/objects/long_format.go b/objects/long_format.go index 867c70d19..7c8d0605b 100644 --- a/objects/long_format.go +++ b/objects/long_format.go @@ -42,6 +42,7 @@ package objects import ( + "errors" "fmt" "math" "math/big" @@ -97,7 +98,7 @@ func intFormat(o Object, spec string) (string, error) { } parsed, err := format.ParseSpec(spec) if err != nil { - return "", fmt.Errorf("ValueError: %w", err) + return "", formatSpecError(err, spec, o.Type().Name) } switch parsed.Type { case 0, 'b', 'c', 'd', 'o', 'x', 'X', 'n': @@ -136,6 +137,23 @@ func unknownPresentationType(presentationType byte, typeName string) error { return fmt.Errorf("ValueError: Unknown format code '\\x%x' for object of type '%s'", uint(presentationType), typeName) } +// formatSpecError converts a format.ParseSpec error into the +// type-aware ValueError CPython raises. The ErrInvalidSpecifier +// sentinel (more than one char remained for the type field) carries no +// type context, so it is rendered here as the full +// "Invalid format specifier '' for object of type ''" +// message using the original spec string and the object's type name. +// Every other parse error keeps its own text behind a "ValueError:" +// prefix. +// +// CPython: Python/formatter_unicode.c:314 PyErr_Format(Invalid format specifier) +func formatSpecError(err error, spec, typeName string) error { + if errors.Is(err, format.ErrInvalidSpecifier) { + return fmt.Errorf("ValueError: Invalid format specifier '%s' for object of type '%s'", spec, typeName) + } + return fmt.Errorf("ValueError: %w", err) +} + // bigIntFromIntLike unwraps an Int or Bool into its underlying big.Int. // Bool embeds Int in gopy, mirroring CPython's PyBool subclassing PyLong. // diff --git a/objects/module.go b/objects/module.go index ac02ba570..a5c3f156f 100644 --- a/objects/module.go +++ b/objects/module.go @@ -533,7 +533,18 @@ func moduleSetattr(o Object, name, value Object) error { } } if value == nil { - return m.dict.DelItem(name) + // Deleting a missing key surfaces as KeyError from the dict; + // generic delattr promotes that to AttributeError. + // + // CPython: Objects/object.c:2026 _PyObject_GenericSetAttrWithDict + if err := m.dict.DelItem(name); err != nil { + if isKeyError(err) { + return fmt.Errorf("AttributeError: '%s' object has no attribute '%s'", + m.Type().Name, attrNameStr(name)) + } + return err + } + return nil } return m.dict.SetItem(name, value) } diff --git a/objects/slice.go b/objects/slice.go index 186aca152..233e4ff18 100644 --- a/objects/slice.go +++ b/objects/slice.go @@ -152,6 +152,15 @@ func NewSlice(start, stop, step Object) *Slice { return s } +// NewSliceFromConst builds a slice from native constant-pool bounds, +// lifting each through wrapConstAttr so scalars, nested tuples, and +// codegen-side const placeholders all surface as Objects. A nil bound +// becomes None. This is the lift path for a slice constant emitted by +// codegen_slice for a fully constant slice. +func NewSliceFromConst(start, stop, step any) *Slice { + return NewSlice(wrapConstAttr(start), wrapConstAttr(stop), wrapConstAttr(step)) +} + // sliceTraverse visits start, stop, and step so the cycle collector // can see references the slice owns. Required for cases like // `o.s = slice(o)` (test_slice.test_cycle) where the slice itself diff --git a/objects/str_bind.go b/objects/str_bind.go index d73a214d3..86e62fb6f 100644 --- a/objects/str_bind.go +++ b/objects/str_bind.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/tamnd/gopy/codecs" + "github.com/tamnd/gopy/format" ) func init() { @@ -739,6 +740,15 @@ func strFormatMethod(args []Object, kwargs map[string]Object) (Object, error) { if err != nil { return nil, err } + // A template with no replacement fields is copied verbatim, which + // for an exact str means returning the receiver itself (do_markup + // aliases input->str). This matches CPython's "{}".format() identity. + // + // CPython: Objects/stringlib/unicode_format.h:836 do_markup + if u, ok := args[0].(*Unicode); ok && args[0].Type() == strType && + !strings.ContainsAny(s, "{}") { + return u, nil + } out, ferr := strFormatExpand(s, args[1:], kwargs) if ferr != nil { return nil, ferr @@ -804,10 +814,35 @@ func strFormatExpandInner(s string, args []Object, kwargs map[string]Object, map return nil, fmt.Errorf("ValueError: Single '{' encountered in format string") } field := s[i+1 : end] - rendered, ferr := strFormatFieldInner(field, args, kwargs, mapObj, auto, depth) + v, spec, ferr := strFormatFieldInner(field, args, kwargs, mapObj, auto, depth) if ferr != nil { return nil, ferr } + // When this field is the last markup token, stop + // overallocating so a single aliased str can be returned by + // identity (mirrors do_markup clearing overallocate once the + // iterator is exhausted). + // + // CPython: Objects/stringlib/unicode_format.h:836 do_markup + lastToken := end+1 >= len(s) + if u, ok := v.(*Unicode); ok && v.Type() == strType && strFormatStrSpecIsIdentity(spec, u.length) { + if lastToken { + w.overallocate = false + } + if err := w.WriteStr(u); err != nil { + return nil, err + } + i = end + 1 + continue + } + // Dispatch through PyObject_Format so each replacement uses + // the argument's own __format__ slot. + // + // CPython: Objects/stringlib/unicode_format.h:1024 output_markup + rendered, rerr := Format(v, spec) + if rerr != nil { + return nil, rerr + } if err := writeBodyChunk(&w, rendered); err != nil { return nil, err } @@ -855,7 +890,15 @@ func strFormatMatchBrace(s string, start int) int { return -1 } -func strFormatFieldInner(field string, args []Object, kwargs map[string]Object, mapObj Object, auto *int, depth int) (string, error) { +// strFormatFieldInner parses one replacement field, resolves its value +// (applying any !s/!r/!a conversion), and expands a nested format spec. +// It returns the value to be formatted together with the final spec +// string; the caller dispatches through PyObject_Format (or, for an +// exact-str value with an identity-preserving spec, aliases the value +// directly so str.format can return its argument by identity). +// +// CPython: Objects/stringlib/unicode_format.h:824 output_markup +func strFormatFieldInner(field string, args []Object, kwargs map[string]Object, mapObj Object, auto *int, depth int) (Object, string, error) { // Find the ':' that separates the field name from the format spec, // skipping any '[...]' subscript components in the name. // @@ -894,17 +937,17 @@ colonDone: } if bang >= 0 { if bang+1 >= len(name) { - return "", fmt.Errorf("ValueError: end of format while looking for conversion specifier") + return nil, "", fmt.Errorf("ValueError: end of format while looking for conversion specifier") } conversion = name[bang+1] if len(name) > bang+2 { - return "", fmt.Errorf("ValueError: expected ':' after conversion specifier") + return nil, "", fmt.Errorf("ValueError: expected ':' after conversion specifier") } name = name[:bang] } v, err := strFormatLookupInner(name, args, kwargs, mapObj, auto) if err != nil { - return "", err + return nil, "", err } switch conversion { case 0: @@ -912,23 +955,23 @@ colonDone: case 's': sv, sErr := StrObject(v) if sErr != nil { - return "", sErr + return nil, "", sErr } v = sv case 'r': rv, rErr := ReprObject(v) if rErr != nil { - return "", rErr + return nil, "", rErr } v = rv case 'a': rv, rErr := Repr(v) if rErr != nil { - return "", rErr + return nil, "", rErr } v = NewStr(asciiEscape(rv)) default: - return "", fmt.Errorf("ValueError: Unknown conversion specifier %c", conversion) + return nil, "", fmt.Errorf("ValueError: Unknown conversion specifier %c", conversion) } // If the format spec itself contains field references (e.g., {:{}}), // recursively expand them before passing to PyObject_Format. @@ -937,15 +980,44 @@ colonDone: if strings.ContainsRune(spec, '{') { expanded, expErr := strFormatExpandInner(spec, args, kwargs, mapObj, auto, depth+1) if expErr != nil { - return "", expErr + return nil, "", expErr } spec = expanded.Value() } - // Dispatch through PyObject_Format so each replacement uses the - // argument's own __format__ slot. - // - // CPython: Objects/stringlib/unicode_format.h:1024 output_markup - return Format(v, spec) + return v, spec, nil +} + +// strFormatStrSpecIsIdentity reports whether spec, applied to an exact +// str of the given codepoint length, performs no transformation, so the +// formatter would just copy the value through (format_string_internal's +// fast path). When true, str.format can alias the argument and return it +// by identity. An unparsable or non-string-typed spec returns false so +// the normal Format path raises the right error. +// +// CPython: Python/formatter_unicode.c:911 format_string_internal (fast path) +func strFormatStrSpecIsIdentity(spec string, length int) bool { + if spec == "" { + return true + } + parsed, err := format.ParseSpec(spec) + if err != nil { + return false + } + if parsed.Type != 0 && parsed.Type != 's' { + return false + } + // Sign / 'z' / '#' / '=' are rejected by the string formatter; let + // the slow path surface those errors rather than aliasing. + if parsed.Sign != 0 || parsed.NoNegZero || parsed.Alt || parsed.Align == '=' { + return false + } + if parsed.Width != -1 && parsed.Width > length { + return false + } + if parsed.Precision != -1 && parsed.Precision < length { + return false + } + return true } // strFormatLookupInner resolves a field name against args/kwargs (or mapObj diff --git a/objects/str_format.go b/objects/str_format.go index 3c08cb177..155723463 100644 --- a/objects/str_format.go +++ b/objects/str_format.go @@ -76,7 +76,7 @@ func unicodeFormat(o Object, spec string) (string, error) { } parsed, err := format.ParseSpec(spec) if err != nil { - return "", fmt.Errorf("ValueError: %w", err) + return "", formatSpecError(err, spec, o.Type().Name) } switch parsed.Type { case 0, 's': diff --git a/objects/str_modulo.go b/objects/str_modulo.go index 284276680..0c04d2e8a 100644 --- a/objects/str_modulo.go +++ b/objects/str_modulo.go @@ -91,27 +91,47 @@ func unicodeModulo(a, b Object) (Object, error) { var out UnicodeWriter out.Init() + // CPython: min_length = fmtcnt + 100; overallocate = 1. The buffer + // is allocated lazily on the first write so a format string that + // produces its result from a single aliased str (e.g. "%s" % s) + // can be returned without a copy. + // + // CPython: Objects/unicodeobject.c:15527 PyUnicode_Format + out.minLength = len(ctx.fmt) + 100 out.overallocate = true - // Pre-size: the literal portion of the format string plus a small - // reserve for substitutions. PrepareInternal grows on demand. - if err := out.PrepareInternal(len(fmtStr.v)+16, 127); err != nil { - return nil, err - } - for ctx.pos < len(ctx.fmt) { + n := len(ctx.fmt) + for ctx.pos < n { ch := ctx.fmt[ctx.pos] if ch != '%' { - if err := out.WriteChar(ch); err != nil { + // Gather the maximal non-'%' run and write it as one + // substring. When the run reaches the end of the format + // string the writer stops overallocating, so a format that + // is a single literal run aliases the format string itself + // (text % () is text). + // + // CPython: Objects/unicodeobject.c:15545 (non-'%' run + overallocate=0) + start := ctx.pos + ctx.pos++ + for ctx.pos < n && ctx.fmt[ctx.pos] != '%' { + ctx.pos++ + } + if ctx.pos == n { + out.overallocate = false + } + if err := out.WriteSubstring(fmtStr, start, ctx.pos); err != nil { return nil, err } - ctx.pos++ continue } ctx.pos++ - if ctx.pos >= len(ctx.fmt) { + if ctx.pos >= n { return nil, fmt.Errorf("ValueError: incomplete format") } if ctx.fmt[ctx.pos] == '%' { + if ctx.pos+1 == n { + out.overallocate = false + } if err := out.WriteChar('%'); err != nil { return nil, err } @@ -222,11 +242,37 @@ widthParse: arg.ch = ctx.fmt[ctx.pos] ctx.pos++ + // CPython clears overallocate once the format string is exhausted + // (fmtcnt == 0), which lets the final write alias its source object. + // + // CPython: Objects/unicodeobject.c:15214 unicode_format_arg_format + if ctx.pos >= len(ctx.fmt) { + out.overallocate = false + } + v, err := nextArg(ctx) if err != nil { return err } + // Fast path for "%s" % str with an exact str argument: write the + // source object straight into the writer (no intermediate copy) so + // it can be aliased and returned by identity. The width/precision + // guard matches unicode_format_arg_output's fast path: padding or a + // truncating precision forces the slow copy below. + // + // CPython: Objects/unicodeobject.c:15231 (Py_NewRef(v)) + // CPython: Objects/unicodeobject.c:15336 unicode_format_arg_output (fast path) + if arg.ch == 's' { + if u, ok := v.(*Unicode); ok && v.Type() == strType { + if (arg.width == -1 || arg.width <= u.length) && + (arg.prec == -1 || arg.prec >= u.length) && + arg.flags&(fmtSign|fmtBlank) == 0 { + return out.WriteStr(u) + } + } + } + body, err := formatBody(&arg, v) if err != nil { return err @@ -470,9 +516,12 @@ func numberAsBigInt(v Object, ch rune) (*big.Int, error) { func formatFloat(v Object, arg *fmtArg) (string, error) { f, ok := asFloat(v) if !ok { + // formatfloat calls PyFloat_AsDouble, which raises this message + // directly rather than the "%c format: ..." wrapper used by the + // integer path (mainformatlong). + // CPython: Objects/floatobject.c:283 PyFloat_AsDouble return "", fmt.Errorf( - "TypeError: %%%c format: a real number is required, not %s", - arg.ch, v.Type().Name) + "TypeError: must be real number, not %s", v.Type().Name) } prec := arg.prec if prec < 0 { diff --git a/parser/lexer/driver_string.go b/parser/lexer/driver_string.go index 689ad80bd..eea64099b 100644 --- a/parser/lexer/driver_string.go +++ b/parser/lexer/driver_string.go @@ -80,6 +80,21 @@ func FromString(src string, mode Mode) *State { ) s.done = eEncoding } + // CPython: Parser/lexer/lexer.c:89 contains_null_bytes + // CPython runs the null-byte scan per-line inside tok_nextc after every + // refill. gopy loads the whole source upfront so the equivalent check is + // a single pre-scan that finds the offending line and records the + // canonical SyntaxError. + if s.err == nil { + if line, ok := firstNullByteLine(buf); ok { + s.lineno = line + s.recordErrorWithText( + "source code cannot contain null bytes", + nthLine(buf, line), + ) + s.done = eSyntax + } + } // exec-input (Py_file_input) gets a trailing newline injection so the // lexer closes the indent stack at EOF via the normal atbol path. // single-input and eval-input do NOT inject: pegen rewrites the first @@ -150,36 +165,40 @@ func FromBytes(src []byte, mode Mode) *State { } } } - // CPython runs _PyTokenizer_ensure_utf8 on the source whenever the - // declared encoding is utf-8 (default, BOM, or explicit cookie). The - // non-utf-8 cookie path skips validation because the codec decode - // already produced canonical utf-8 above. + // CPython interleaves both diagnostics: tok_nextc reads a line, the + // codec/ensure_utf8 step rejects an undecodable line, and + // contains_null_bytes rejects an embedded NUL, all on the same line + // before the next line is read. So the fault on the earlier line wins, + // and a NUL beats a non-UTF-8 byte sitting on the same line. gopy + // buffers the whole source upfront, so compute both candidate lines + // and report whichever the per-line tokenizer would have hit first. // // CPython: Parser/tokenizer/string_tokenizer.c:108 ensure_utf8 call + // CPython: Parser/lexer/lexer.c:89 contains_null_bytes + utf8Line, utf8Bad, utf8OK := 0, byte(0), false if s.err == nil && !nonUTF8Cookie { if line, bad, ok := ValidateUTF8(src); !ok { - s.lineno = line - s.recordErrorWithText( - nonUTF8ErrorMessage(bad, line), - nthLine(src, line), - ) - s.done = eEncoding + utf8Line, utf8Bad, utf8OK = line, bad, true } } - // CPython: Parser/lexer/lexer.c:89 contains_null_bytes - // CPython runs the null-byte scan per-line inside tok_nextc after - // every refill. gopy loads the whole source upfront in FromBytes so - // the equivalent check is a single pre-scan that finds the offending - // line and records the canonical SyntaxError. + nullLine, nullOK := 0, false if s.err == nil { - if line, ok := firstNullByteLine(src); ok { - s.lineno = line - s.recordErrorWithText( - "source code cannot contain null bytes", - nthLine(src, line), - ) - s.done = eSyntax - } + nullLine, nullOK = firstNullByteLine(src) + } + if nullOK && (!utf8OK || nullLine <= utf8Line) { + s.lineno = nullLine + s.recordErrorWithText( + "source code cannot contain null bytes", + nthLine(src, nullLine), + ) + s.done = eSyntax + } else if utf8OK { + s.lineno = utf8Line + s.recordErrorWithText( + nonUTF8ErrorMessage(utf8Bad, utf8Line), + nthLine(src, utf8Line), + ) + s.done = eEncoding } // CPython: Parser/pegen.c:1048 exec_input = start_rule == Py_file_input src = TranslateNewlines(src, mode == ModeFile) diff --git a/parser/parser.go b/parser/parser.go index 9248a1042..f42c8cb08 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -286,6 +286,26 @@ func runParse(st *lexer.State, mode Mode, flags int, featureVersion ...int) (mod if err != nil { return nil, err } + // Py_single_input mode rejects source that carries a second statement + // past the one the interactive grammar matched. CPython runs this check + // in _PyPegen_run_parser after a successful parse, scanning the raw + // source past the tokenizer cursor; gopy walks the residual token stream. + // + // CPython: Parser/pegen.c:754 bad_single_statement + if mode == ModeSingle && p.BadSingleStatement() { + p.RaiseMultipleStatements() + if e := p.PinnedError(); e != nil { + if e.Text == "" { + if ts := p.Tokenizer(); ts != nil { + e.Text = ts.SourceLine(e.Pos.Lineno) + } + } + if e.Filename == "" { + e.Filename = st.Filename() + } + return nil, e + } + } if mod, ok := node.(ast.Mod); ok { return mod, nil } diff --git a/parser/pegen/errors.go b/parser/pegen/errors.go index c5eaa8b92..1c2285afc 100644 --- a/parser/pegen/errors.go +++ b/parser/pegen/errors.go @@ -34,6 +34,17 @@ func (p *Parser) RaiseSyntaxError(format string, args ...any) { p.recordError(perrors.KindSyntax, pos, fmt.Sprintf(format, args...)) } +// RaiseMultipleStatements pins the "multiple statements found while +// compiling a single statement" SyntaxError that Py_single_input mode +// raises when source past the parsed statement carries non-trivial +// tokens. Kept here (not in the generated table) so parser_gen.go need +// not import the parser/errors message catalog. +// +// CPython: Parser/pegen.c:754 bad_single_statement +func (p *Parser) RaiseMultipleStatements() { + p.RaiseSyntaxError(perrors.MsgMultipleStmtsInteract) +} + // RaiseSyntaxErrorKnownLocation pins a SyntaxError at a caller-given // span. Used by action helpers that already know where they want the // caret to land (invalid LHS, "did you mean :=", paren mismatch). diff --git a/parser/pegen/parser.go b/parser/pegen/parser.go index 685fda621..9ac26f3ee 100644 --- a/parser/pegen/parser.go +++ b/parser/pegen/parser.go @@ -708,3 +708,32 @@ func (p *Parser) IsEndOfSource() bool { func (p *Parser) AllowIncompleteInput() bool { return p.flags&FlagAllowIncompleteInput != 0 } + +// BadSingleStatement reports whether a single-input parse left a second +// statement unconsumed. CPython scans the raw source past the tokenizer +// cursor for any non-whitespace, non-comment character; gopy walks the +// token stream from the current mark instead, skipping the trivia tokens +// (NEWLINE / NL / INDENT / DEDENT / COMMENT / ENDMARKER) that a complete +// single statement legitimately leaves behind. Any other token means a +// second statement followed, so "1\n2" and "def f(): pass" in single +// mode raise SyntaxError just as the interactive compiler requires. +// +// CPython: Parser/pegen.c:754 bad_single_statement +func (p *Parser) BadSingleStatement() bool { + mark := p.mark + for { + t := p.Peek() + if t == nil || t.Type == token.ENDMARKER { + p.mark = mark + return false + } + switch t.Type { + case token.NEWLINE, token.NL, token.INDENT, token.DEDENT, + token.COMMENT: + p.mark++ + default: + p.mark = mark + return true + } + } +} diff --git a/pystrconv/dtoa.go b/pystrconv/dtoa.go index a7d00164d..89a4e62e4 100644 --- a/pystrconv/dtoa.go +++ b/pystrconv/dtoa.go @@ -76,7 +76,7 @@ func FormatFloat(f float64, code byte, precision int, flags FloatFormatFlag) str func formatFloatShort(f float64, code byte, precision int, flags FloatFormatFlag) string { var sign string switch { - case math.Signbit(f) && !(flags&FlagNoNegZero != 0 && f == 0): + case math.Signbit(f): sign = "-" case flags&FlagAlwaysSign != 0: sign = "+" @@ -108,6 +108,24 @@ func formatFloatShort(f float64, code byte, precision int, flags FloatFormatFlag digits, decpt := shortestDigits(f, code, precision) + // Py_DTSF_NO_NEG_0 (the 'z' format flag) coerces a value that + // rounds to zero back to a positive zero. The sign was taken from + // the original (unrounded) value above, so a small negative input + // like -0.01 at .1f keeps the '-' here even though its digits are + // all zero; clear it once the rounded digit string is known. + // + // CPython: Python/pystrtod.c:1059 format_float_short (NO_NEG_0) + if flags&FlagNoNegZero != 0 && sign == "-" && allZeroDigits(digits) { + switch { + case flags&FlagAlwaysSign != 0: + sign = "+" + case flags&FlagSpaceSign != 0: + sign = " " + default: + sign = "" + } + } + body := layoutFloat(digits, decpt, code, precision, flags) if addPercent { body += "%" @@ -149,6 +167,17 @@ func shortestDigits(f float64, code byte, precision int) (digits string, decpt i buf = strconv.AppendFloat(nil, abs, 'e', precision-1, 64) mantissa, exp = splitScientific(buf) } + // dtoa mode 2 (which 'g' uses) returns the minimal digit string: + // trailing zeros produced by the fixed-precision re-render above + // are not significant and must be dropped. The '#'/ALT path + // re-pads them in layoutFloat via vdigitsEnd, so stripping here + // is safe for both forms. + // + // CPython: Python/pystrtod.c:1009 format_float_short (mode 2 digits) + mantissa = strings.TrimRight(mantissa, "0") + if mantissa == "" { + mantissa = "0" + } return mantissa, exp + 1 case 'e': buf := strconv.AppendFloat(nil, abs, 'e', precision, 64) @@ -273,6 +302,17 @@ func layoutFloat(digits string, decpt int, code byte, precision int, flags Float return b.String() } +// allZeroDigits reports whether the digit string is composed entirely +// of '0' (i.e. the value rounded to zero). +func allZeroDigits(digits string) bool { + for i := 0; i < len(digits); i++ { + if digits[i] != '0' { + return false + } + } + return true +} + func writeDigitsClipped(b *strings.Builder, digits string, start, end int) { for i := start; i < end; i++ { if i < 0 || i >= len(digits) { diff --git a/pythonrun/runstring.go b/pythonrun/runstring.go index c3dca72cc..2cff896e1 100644 --- a/pythonrun/runstring.go +++ b/pythonrun/runstring.go @@ -38,11 +38,11 @@ func RunString(ts *state.Thread, src, filename string, mode parser.Mode, globals if err != nil { return nil, err } - cco, err := compile.Compile(mod, filename, 0) + cco, err := compile.Compile(mod, filename, -1) if err != nil { return nil, err } - return vm.EvalCode(ts, liftCode(cco), globals, locals) + return vm.EvalCode(ts, liftTopCode(cco), globals, locals) } // RunBytes is the bytes-input variant of RunString. The PEP 263 @@ -62,11 +62,11 @@ func RunBytes(ts *state.Thread, src []byte, filename string, mode parser.Mode, g if err != nil { return nil, err } - cco, err := compile.Compile(mod, filename, 0) + cco, err := compile.Compile(mod, filename, -1) if err != nil { return nil, err } - return vm.EvalCode(ts, liftCode(cco), globals, locals) + return vm.EvalCode(ts, liftTopCode(cco), globals, locals) } // RunSimpleString parses, compiles, and runs command as a Python @@ -143,6 +143,17 @@ func printRunError(ts *state.Thread, err error, w io.Writer) int { return 1 } +// liftTopCode lifts a freshly compiled top-level code unit and runs the +// per-compile constant merge so co_consts / co_linetable / co_filename +// are shared across the whole tree, matching CPython's const_cache. +// +// CPython: Python/compile.c:1707 const_cache lifetime +func liftTopCode(c *compile.Code) *objects.Code { + out := liftCode(c) + objects.InternCodeConstants(out) + return out +} + // liftCode adapts compile.Code into objects.Code. The two structs // will collapse once spec 1687 retires compile.Code. func liftCode(c *compile.Code) *objects.Code { @@ -200,6 +211,8 @@ func liftRunConst(v any) any { items[i] = liftRunConst(raw) } return items + case *compile.ConstSlice: + return objects.NewSliceFromConst(x.Start, x.Stop, x.Step) } return v } diff --git a/specialize/deopt.go b/specialize/deopt.go index 865196277..aba003dcb 100644 --- a/specialize/deopt.go +++ b/specialize/deopt.go @@ -17,7 +17,10 @@ package specialize -import "github.com/tamnd/gopy/compile" +import ( + "github.com/tamnd/gopy/compile" + "github.com/tamnd/gopy/objects" +) // CacheCount returns the number of trailing codeunits reserved as // inline cache after op. Zero means op carries no cache. Source of @@ -50,6 +53,12 @@ func init() { for variant, parent := range DeoptParent { deoptTable[variant] = parent } + // Wire the co_code getter so it returns the deoptimized snapshot. + // objects cannot import specialize (specialize imports objects), so + // the dependency is inverted through this hook. + // + // CPython: Objects/codeobject.c:2310 _PyCode_GetCode + objects.CodeDeoptHook = DeoptCode } // Deopt returns the adaptive parent of op. For an unspecialized diff --git a/state/state.go b/state/state.go index 9e87e199b..7bdc072e1 100644 --- a/state/state.go +++ b/state/state.go @@ -163,6 +163,15 @@ type Thread struct { // CPython: Include/cpython/pystate.h:128 tracing Tracing int + // WhatEvent is the monitoring event currently being delivered to a + // trace/profile callback, or -1 when no callback is running. The + // f_lineno setter consults it to decide whether a line jump is + // permitted: jumps are only allowed from a 'line' (and a handful of + // related) event, and never when not tracing. + // + // CPython: Include/cpython/pystate.h:130 what_event + WhatEvent int + // CoroutineOriginTrackingDepth is how many traceback frames to // capture on coroutine creation, set by // sys.set_coroutine_origin_tracking_depth. Depth 0 disables @@ -365,7 +374,7 @@ func (r *Runtime) DropInterpreters() { // // CPython: Python/pystate.c:L915 PyThreadState_New func (i *Interpreter) AttachThread() *Thread { - t := &Thread{interp: i, id: threadIDCounter.Add(1)} + t := &Thread{interp: i, id: threadIDCounter.Add(1), WhatEvent: -1} t.exc.Store(excHolder{}) i.threads = append(i.threads, t) return t diff --git a/stdlib/test/dis_module.py b/stdlib/test/dis_module.py new file mode 100644 index 000000000..afbf600fd --- /dev/null +++ b/stdlib/test/dis_module.py @@ -0,0 +1,5 @@ + +# A simple module for testing the dis module. + +def f(): pass +def g(): pass diff --git a/vm/dispatch.go b/vm/dispatch.go index 12282a97a..a8d2a6b30 100644 --- a/vm/dispatch.go +++ b/vm/dispatch.go @@ -49,6 +49,18 @@ func (e *evalState) dispatch(op compile.Opcode, oparg uint32) (next int, err err if err != nil { return 0, err } + if e.f.LinenoJumped { + // A trace callback set f_lineno during the LINE event: resume at + // the relocated instruction pointer without executing the opcode + // the INSTRUMENTED_LINE marker was hiding. The flag is the explicit + // jump signal; a bare InstrPtr-before/after comparison would also + // trip on the EXTENDED_ARG-prefix advance fetchExtended performs. + // + // CPython: Python/bytecodes.c INSTRUMENTED_LINE checks + // frame->instr_ptr != this_instr after the line event fires. + e.f.LinenoJumped = false + return e.f.InstrPtr, nil + } // Specializer routing: only Quickened code carries inline-cache // counters and specialized variants; non-Quickened code (raw // compile output before specialize.Quicken) skips the entire diff --git a/vm/eval.go b/vm/eval.go index b01c9b5c5..0fbd5807e 100644 --- a/vm/eval.go +++ b/vm/eval.go @@ -431,6 +431,21 @@ func (e *evalState) run() (objects.Object, error) { if e.f.Owner != frame.OwnedByGenerator { e.f.DropStack(e.f.StackTop) } + // A reraiseError suppresses attachFrameTraceback only for the + // frame that executed RERAISE (its goto exception_unwind skips + // the `error` label's PyTraceBack_Here). Once that frame has no + // handler and the exception propagates to the caller, CPython's + // caller-side error label DOES call PyTraceBack_Here for the + // caller frame. Unwrap the marker here so the next frame up + // attaches its own tb entry instead of inheriting the skip + // (otherwise an exception escaping exec()/eval() loses every + // caller frame above the reraising one). + // + // CPython: Python/ceval.c error label (PyTraceBack_Here per frame) + var rr *reraiseError + if errors.As(err, &rr) { + err = excSentinel(rr.exc) + } return nil, err } e.f.InstrPtr = next diff --git a/vm/eval_call.go b/vm/eval_call.go index 682394593..305f5083c 100644 --- a/vm/eval_call.go +++ b/vm/eval_call.go @@ -25,15 +25,14 @@ import ( // keywordCandidates lists the parameter names eligible for keyword // binding so UnexpectedKeywordError can offer a "Did you mean 'X'?" // suggestion. It mirrors CPython's co_varnames[posonlyargcount : -// total_args]: positional-only slots and the *args slot are skipped. +// total_args]: positional-only slots are skipped. The *args / **kwargs +// slots follow the kw-only window (kwWindow = argcount + kwonlyargcount) +// so they fall outside the [nposonly, kwWindow) range already. // // CPython: Python/ceval.c:1792 (possible_keywords list) -func keywordCandidates(co *objects.Code, nposonly, npos, kwWindow int, hasVarargs bool) []string { +func keywordCandidates(co *objects.Code, nposonly, kwWindow int) []string { out := make([]string, 0, kwWindow) for i := nposonly; i < kwWindow; i++ { - if i == npos && hasVarargs { - continue - } out = append(out, co.Varnames[i]) } return out @@ -245,6 +244,12 @@ func init() { // // CPython: Python/legacy_tracing.c:159 _PyEval_SetOpcodeTrace objects.SetOpcodeTraceHook = setOpcodeTraceHook + // Back frame.f_lineno's setter (the debugger line jump). objects/ + // cannot reach the mark_stacks / instruction-pointer machinery, so + // it routes through this hook. + // + // CPython: Objects/frameobject.c:1640 frame_lineno_set_impl + objects.SetLinenoHook = frameLinenoSetHook // Expose the same hook to module/sys for sys._getframe(). // // CPython: Python/sysmodule.c:1180 sys__getframe_impl @@ -544,14 +549,18 @@ func callPyFunction(o objects.Object, args []objects.Object, kwargs map[string]o f.SetLocal(i, stackref.FromObjectNew(args[i])) } // *args: pack any extra positionals into a tuple at the varargs slot. + // The varargs slot follows the kw-only slots, so it sits at + // npos+nkwonly (co_varnames order: posonly, posorkw, kwonly, *args, + // **kwargs). if hasVarargs { extra := args[bound:] items := make([]objects.Object, len(extra)) copy(items, extra) - f.SetLocal(npos, stackref.FromObject(objects.NewTuple(items))) + f.SetLocal(npos+nkwonly, stackref.FromObject(objects.NewTuple(items))) } // **kwargs: collect unknown keyword args here. Allocate eagerly so - // the slot is bound even when no keywords are passed. + // the slot is bound even when no keywords are passed. It sits after + // the varargs slot when one is present. var kwSlot int var kwDict *objects.Dict if hasVarkw { @@ -565,16 +574,13 @@ func callPyFunction(o objects.Object, args []objects.Object, kwargs map[string]o // Keyword bind: scan the positional + kw-only window for a name // match. Positional-only slots [0..nposonly) are not eligible for // keyword binding; collisions route to **kwargs when present or - // surface as positional_only_passed_as_keyword. With *args, the - // varargs slot sits at index npos so the kw-only slots shift to - // [npos+1 .. npos+1+nkwonly). + // surface as positional_only_passed_as_keyword. The kw-eligible + // slots are [nposonly .. npos+nkwonly); the varargs / varkw slots + // follow and are never keyword targets. // // CPython: Python/ceval.c:1546 positional_only_passed_as_keyword // CPython: Objects/call.c _PyEval_BindArguments keyword loop kwWindow := npos + nkwonly - if hasVarargs { - kwWindow++ - } var posonlyAsKw []string for k, v := range kwargs { idx := -1 @@ -582,9 +588,6 @@ func callPyFunction(o objects.Object, args []objects.Object, kwargs map[string]o if i < nposonly { continue } - if i == npos && hasVarargs { - continue - } if co.Varnames[i] == k { idx = i break @@ -610,7 +613,7 @@ func callPyFunction(o objects.Object, args []objects.Object, kwargs map[string]o continue } } - return nil, objects.UnexpectedKeywordError(qualname, k, keywordCandidates(co, nposonly, npos, kwWindow, hasVarargs)) + return nil, objects.UnexpectedKeywordError(qualname, k, keywordCandidates(co, nposonly, kwWindow)) } if !f.LocalAt(idx).IsNull() { return nil, objects.MultipleValuesForArgumentError(qualname, k) @@ -651,12 +654,10 @@ func callPyFunction(o objects.Object, args []objects.Object, kwargs map[string]o } } } - // Keyword-only defaults fill any unbound kw-only slots. + // Keyword-only defaults fill any unbound kw-only slots. The kw-only + // slots sit at [npos, npos+nkwonly), before any *args / **kwargs. if fn.KwDefaults != nil && nkwonly > 0 { base := npos - if hasVarargs { - base++ - } for i := 0; i < nkwonly; i++ { slot := base + i if !f.LocalAt(slot).IsNull() { @@ -682,9 +683,6 @@ func callPyFunction(o objects.Object, args []objects.Object, kwargs map[string]o return nil, objects.MissingArgumentsError(qualname, "positional", missingPos) } kwOnlyBase := npos - if hasVarargs { - kwOnlyBase++ - } var missingKw []string for i := 0; i < nkwonly; i++ { slot := kwOnlyBase + i @@ -781,7 +779,6 @@ func pyFunctionVectorcall(o objects.Object, args []objects.Object, nargsf uint, npos := co.Argcount nkwonly := co.KwonlyArgcount nposonly := co.PosonlyArgcount - hasVarargs := co.Flags&int(0x04) != 0 hasVarkw := co.Flags&int(0x08) != 0 qualname := fn.Qualname if qualname == "" { @@ -799,9 +796,6 @@ func pyFunctionVectorcall(o objects.Object, args []objects.Object, nargsf uint, nkw := kwnames.Len() kwargs = make(map[string]objects.Object, nkw) kwWindow := npos + nkwonly - if hasVarargs { - kwWindow++ - } for i := 0; i < nkw; i++ { kwname := kwnames.Item(i) val := args[nposArgs+i] @@ -823,9 +817,6 @@ func pyFunctionVectorcall(o objects.Object, args []objects.Object, nargsf uint, if j < nposonly { continue } - if j == npos && hasVarargs { - continue - } eq, err := objects.RichCmpBool(kwname, objects.NewStr(co.Varnames[j]), objects.CompareEQ) if err != nil { return nil, err @@ -844,7 +835,7 @@ func pyFunctionVectorcall(o objects.Object, args []objects.Object, nargsf uint, if !hasVarkw { // Best-effort string repr for error message. ks, _ := objects.Str(kwname) - return nil, objects.UnexpectedKeywordError(qualname, ks, keywordCandidates(co, nposonly, npos, kwWindow, hasVarargs)) + return nil, objects.UnexpectedKeywordError(qualname, ks, keywordCandidates(co, nposonly, kwWindow)) } rawKwEntries = append(rawKwEntries, rawKwEntry{key: kwname, val: val}) } @@ -916,7 +907,7 @@ func callPyFunctionRaw(fn *objects.Function, co *objects.Code, posArgs []objects objects.Incref(it) } } - f.SetLocal(npos, stackref.FromObject(objects.NewTuple(items))) + f.SetLocal(npos+nkwonly, stackref.FromObject(objects.NewTuple(items))) } var kwSlot int var kwDict *objects.Dict @@ -929,9 +920,6 @@ func callPyFunctionRaw(fn *objects.Function, co *objects.Code, posArgs []objects f.SetLocal(kwSlot, stackref.FromObject(kwDict)) } kwWindow := npos + nkwonly - if hasVarargs { - kwWindow++ - } // Iterate kwargs in kwOrder (preserving caller insertion order) when // available, otherwise fall back to Go map iteration. kwOrder is // always non-nil when called from pyFunctionVectorcall. @@ -954,9 +942,6 @@ func callPyFunctionRaw(fn *objects.Function, co *objects.Code, posArgs []objects if i < nposonly { continue } - if i == npos && hasVarargs { - continue - } if co.Varnames[i] == k { idx = i break @@ -982,7 +967,7 @@ func callPyFunctionRaw(fn *objects.Function, co *objects.Code, posArgs []objects continue } } - return nil, objects.UnexpectedKeywordError(qualname, k, keywordCandidates(co, nposonly, npos, kwWindow, hasVarargs)) + return nil, objects.UnexpectedKeywordError(qualname, k, keywordCandidates(co, nposonly, kwWindow)) } if !f.LocalAt(idx).IsNull() { return nil, objects.MultipleValuesForArgumentError(qualname, k) @@ -1032,9 +1017,6 @@ func callPyFunctionRaw(fn *objects.Function, co *objects.Code, posArgs []objects } if fn.KwDefaults != nil && nkwonly > 0 { base := npos - if hasVarargs { - base++ - } for i := 0; i < nkwonly; i++ { slot := base + i if !f.LocalAt(slot).IsNull() { @@ -1057,9 +1039,6 @@ func callPyFunctionRaw(fn *objects.Function, co *objects.Code, posArgs []objects return nil, objects.MissingArgumentsError(qualname, "positional", missingPos) } kwOnlyBase := npos - if hasVarargs { - kwOnlyBase++ - } var missingKw []string for i := 0; i < nkwonly; i++ { slot := kwOnlyBase + i @@ -1081,9 +1060,6 @@ func callPyFunctionRaw(fn *objects.Function, co *objects.Code, posArgs []objects } kwonlyGiven := 0 kwOnlyBaseInner := npos - if hasVarargs { - kwOnlyBaseInner++ - } for i := 0; i < nkwonly; i++ { if !f.LocalAt(kwOnlyBaseInner + i).IsNull() { name := co.Varnames[kwOnlyBaseInner+i] diff --git a/vm/eval_helpers.go b/vm/eval_helpers.go index 866844072..588e045bc 100644 --- a/vm/eval_helpers.go +++ b/vm/eval_helpers.go @@ -684,7 +684,13 @@ func sliceContainer(container, start, stop objects.Object) (objects.Object, erro sl := objects.NewSlice(start, stop, nil) return objects.StrGetSlice(c, sl) } - return nil, errors.New("TypeError: BINARY_SLICE: unsupported container type '" + container.Type().Name + "'") + // CPython BINARY_SLICE builds a slice object and defers to + // PyObject_GetItem for any container without a fast path, so + // memoryview / bytes / bytearray and user types reach their + // mp_subscript here instead of hitting an error. + // + // CPython: Python/bytecodes.c BINARY_SLICE (_PyBuildSlice_ConsumeRefs + PyObject_GetItem) + return objects.GetItem(container, objects.NewSlice(start, stop, nil)) } func storeSlice(container, start, stop, value objects.Object) error { diff --git a/vm/eval_simple.go b/vm/eval_simple.go index b13233205..823aa5139 100644 --- a/vm/eval_simple.go +++ b/vm/eval_simple.go @@ -57,6 +57,8 @@ func init() { items[i] = item } return objects.NewTuple(items), true + case *compile.ConstSlice: + return objects.NewSliceFromConst(x.Start, x.Stop, x.Step), true case ast.EllipsisType: return objects.Ellipsis(), true case ast.FrozenSet: @@ -144,6 +146,8 @@ func liftConst(v any) any { items[i] = liftConst(raw) } return items + case *compile.ConstSlice: + return objects.NewSliceFromConst(x.Start, x.Stop, x.Step) } return v } @@ -169,7 +173,7 @@ func wrapConst(v any) (objects.Object, error) { case float64: return objects.NewFloat(x), nil case string: - return objects.NewStr(x), nil + return objects.WrapConstStr(x), nil case *compile.ConstTuple: items := make([]objects.Object, len(x.Values)) for i, raw := range x.Values { @@ -193,6 +197,8 @@ func wrapConst(v any) (objects.Object, error) { items[i] = item } return objects.NewTuple(items), nil + case *compile.ConstSlice: + return objects.NewSliceFromConst(x.Start, x.Stop, x.Step), nil case *compile.Code: return liftNestedCode(x), nil case []byte: @@ -561,9 +567,9 @@ func (e *evalState) trySimple(op compile.Opcode, oparg uint32) (next int, ok boo case compile.SET_FUNCTION_ATTRIBUTE: // Stack: [func, attr]. oparg's bit identifies the attribute: - // 0x01 = defaults tuple, 0x02 = kwdefaults dict, 0x04 = annotations, - // 0x08 = closure tuple. v0.6 stores the ones we know about and - // ignores the rest. + // 0x01 = defaults tuple, 0x02 = kwdefaults dict, 0x04 = annotations + // dict, 0x08 = closure tuple, 0x10 = __annotate__ callable. CPython + // indexes _Py_FunctionAttributeOffsets[oparg] to the matching slot. // // CPython: Python/bytecodes.c SET_FUNCTION_ATTRIBUTE fnObj := e.popObject() @@ -582,7 +588,14 @@ func (e *evalState) trySimple(op compile.Opcode, oparg uint32) (next int, ok boo fn.KwDefaults = d } case 0x04: - // CPython: Python/bytecodes.c SET_FUNCTION_ATTRIBUTE 0x04 + // MAKE_FUNCTION_ANNOTATIONS: a literal __annotations__ dict. + // CPython: Python/bytecodes.c:4966 SET_FUNCTION_ATTRIBUTE + if d, ok := attr.(*objects.Dict); ok { + fn.Annotations = d + } + case 0x10: + // MAKE_FUNCTION_ANNOTATE: the PEP 649 __annotate__ callable. + // CPython: Python/bytecodes.c:4966 SET_FUNCTION_ATTRIBUTE fn.Annotate = attr fn.Annotations = nil // gh-137814: fix the qualname of the annotation function to diff --git a/vm/exception_table_parity_test.go b/vm/exception_table_parity_test.go index a3910cfdc..23fe80b49 100644 --- a/vm/exception_table_parity_test.go +++ b/vm/exception_table_parity_test.go @@ -182,10 +182,13 @@ func walkExcEntries(tab []byte) []excEntry { return out } pos += n + // Mirror findExcHandler: the table stores code-unit offsets, + // scaled to bytes (two bytes per code unit) so they line up + // with the VM's byte-based instruction pointer. out = append(out, excEntry{ - start: start, - end: start + size, - target: target, + start: start * 2, + end: (start + size) * 2, + target: target * 2, depth: depthLasti >> 1, preserveLasti: depthLasti&1 != 0, }) diff --git a/vm/exctable.go b/vm/exctable.go index 1473fb24d..c366fdc45 100644 --- a/vm/exctable.go +++ b/vm/exctable.go @@ -2,7 +2,10 @@ // emits one entry per handler region as four varints: start, size, // target, depth_lasti. The first byte of each entry has bit 7 // (entryStartBit, 0x80) set; varints chain via the continuation bit -// (0x40). Both fields are byte offsets into the bytecode blob. +// (0x40). The serialized start/size/target fields are code-unit +// offsets (matching CPython's co_exceptiontable format); findExcHandler +// scales them to byte offsets so they line up with the VM's byte-based +// instruction pointer. // // CPython: Python/ceval.c:L1815 get_exception_handler // CPython: Objects/exception_handling_notes.txt format spec @@ -83,6 +86,13 @@ func findExcHandler(tab []byte, pc int) (excEntry, bool) { } pos += n + // The table stores code-unit offsets; the VM tracks the + // instruction pointer in bytes (two bytes per code unit), so + // scale here before comparing against pc. + start *= 2 + size *= 2 + target *= 2 + end := start + size if pc >= start && pc < end { return excEntry{ diff --git a/vm/exctable_test.go b/vm/exctable_test.go index d39698029..6a6ef5901 100644 --- a/vm/exctable_test.go +++ b/vm/exctable_test.go @@ -59,23 +59,26 @@ func TestReadExcVarintTruncated(t *testing.T) { } func TestFindExcHandlerHit(t *testing.T) { - // One entry: start=4, size=6 (covers [4,10)), target=20, depth=2, preserveLasti=0 + // Encoded fields are code-unit offsets: start=4, size=6 (covers + // code units [4,10)), target=20, depth=2. findExcHandler scales + // these to bytes, so the live range is [8,20) and target=40. tab := encEntry(nil, 4, 6, 20, 2<<1) - for _, pc := range []int{4, 5, 9} { + for _, pc := range []int{8, 9, 19} { entry, ok := findExcHandler(tab, pc) if !ok { t.Errorf("pc=%d: expected hit", pc) continue } - if entry.start != 4 || entry.end != 10 || entry.target != 20 || entry.depth != 2 || entry.preserveLasti { + if entry.start != 8 || entry.end != 20 || entry.target != 40 || entry.depth != 2 || entry.preserveLasti { t.Errorf("pc=%d: got %+v", pc, entry) } } } func TestFindExcHandlerMiss(t *testing.T) { + // Code units [4,10) scale to bytes [8,20); everything outside misses. tab := encEntry(nil, 4, 6, 20, 0) - for _, pc := range []int{0, 3, 10, 100} { + for _, pc := range []int{0, 7, 20, 100} { if _, ok := findExcHandler(tab, pc); ok { t.Errorf("pc=%d: expected miss", pc) } @@ -85,11 +88,12 @@ func TestFindExcHandlerMiss(t *testing.T) { func TestFindExcHandlerSecondEntry(t *testing.T) { tab := encEntry(nil, 0, 4, 100, 0) tab = encEntry(tab, 8, 4, 200, (3<<1)|1) - entry, ok := findExcHandler(tab, 9) + // Second entry covers code units [8,12) -> bytes [16,24). + entry, ok := findExcHandler(tab, 18) if !ok { t.Fatal("expected hit on second entry") } - if entry.target != 200 || entry.depth != 3 || !entry.preserveLasti { + if entry.target != 400 || entry.depth != 3 || !entry.preserveLasti { t.Errorf("got %+v", entry) } } @@ -101,13 +105,13 @@ func TestFindExcHandlerEmpty(t *testing.T) { } func TestFindExcHandlerLargeOffsets(t *testing.T) { - // Force multi-byte varints. + // Force multi-byte varints. Encoded code units scale to bytes (2x). tab := encEntry(nil, 1<<14, 1<<10, 1<<16, 0) - entry, ok := findExcHandler(tab, 1<<14+5) + entry, ok := findExcHandler(tab, 2<<14+5) if !ok { t.Fatal("expected hit") } - if entry.start != 1<<14 || entry.end != 1<<14+1<<10 || entry.target != 1<<16 { + if entry.start != 2<<14 || entry.end != 2<<14+2<<10 || entry.target != 2<<16 { t.Errorf("got %+v", entry) } } diff --git a/vm/frame_setlineno.go b/vm/frame_setlineno.go new file mode 100644 index 000000000..5a0a8df3e --- /dev/null +++ b/vm/frame_setlineno.go @@ -0,0 +1,550 @@ +// Settable frame.f_lineno: the debugger line-jump feature. A trace +// function assigns frame.f_lineno to relocate the running frame's +// instruction pointer to a chosen source line. The jump is only legal +// when the evaluation stack at the target offset is compatible with the +// stack at the current offset, so the bulk of this file models the +// abstract stack across the whole code object (mark_stacks) and then +// finds the deepest compatible target on the requested line. +// +// CPython: Objects/frameobject.c:1640 frame_lineno_set_impl and the +// helper block above it (mark_stacks, compatible_stack, marklines, +// first_line_not_before). + +package vm + +import ( + "fmt" + + warnings "github.com/tamnd/gopy/module/_warnings" + + "github.com/tamnd/gopy/compile" + "github.com/tamnd/gopy/errors" + "github.com/tamnd/gopy/frame" + "github.com/tamnd/gopy/monitor" + "github.com/tamnd/gopy/objects" + "github.com/tamnd/gopy/stackref" + "github.com/tamnd/gopy/state" +) + +// Abstract stack-entry kinds. The evaluation stack is modelled as a +// 64-bit integer of 3-bit blocks, one per live value, so jump +// compatibility reduces to integer comparison. +// +// CPython: Objects/frameobject.c:1166 typedef enum kind +type stackKind int64 + +const ( + kindIterator stackKind = 1 + kindExcept stackKind = 2 + kindObject stackKind = 3 + kindNull stackKind = 4 + kindLasti stackKind = 5 +) + +const ( + bitsPerBlock = 3 + stackMask = (1 << bitsPerBlock) - 1 + maxStackEntries = 63 / bitsPerBlock + willOverflow = int64(1) << uint((maxStackEntries-1)*bitsPerBlock) + + stackUninitialized int64 = -2 + stackOverflowed int64 = -1 + stackEmpty int64 = 0 +) + +// compatibleKind reports whether a value of kind from can satisfy a +// slot expecting kind to. +// +// CPython: Objects/frameobject.c:1175 compatible_kind +func compatibleKind(from, to stackKind) bool { + if to == 0 { + return false + } + if to == kindObject { + return from != kindNull + } + if to == kindNull { + return true + } + return from == to +} + +// CPython: Objects/frameobject.c:1199 push_value +func pushValue(stack int64, kind stackKind) int64 { + if uint64(stack) >= uint64(willOverflow) { + return stackOverflowed + } + return (stack << bitsPerBlock) | int64(kind) +} + +// CPython: Objects/frameobject.c:1209 pop_value (arithmetic right shift) +func popValue(stack int64) int64 { + return stack >> bitsPerBlock +} + +// CPython: Objects/frameobject.c:1217 top_of_stack +func topOfStack(stack int64) stackKind { + return stackKind(stack & stackMask) +} + +// CPython: Objects/frameobject.c:1223 peek +func peekStack(stack int64, n int) stackKind { + return stackKind((stack >> uint(bitsPerBlock*(n-1))) & stackMask) +} + +// CPython: Objects/frameobject.c:1230 stack_swap +func stackSwap(stack int64, n int) int64 { + toSwap := peekStack(stack, n) + top := topOfStack(stack) + shift := uint(bitsPerBlock * (n - 1)) + replacedLow := (stack &^ (int64(stackMask) << shift)) | (int64(top) << shift) + replacedTop := (replacedLow &^ int64(stackMask)) | int64(toSwap) + return replacedTop +} + +// CPython: Objects/frameobject.c:1243 pop_to_level +func popToLevel(stack int64, level int) int64 { + if level == 0 { + return stackEmpty + } + maxItem := int64((1 << bitsPerBlock) - 1) + levelMaxStack := maxItem << uint((level-1)*bitsPerBlock) + for stack > levelMaxStack { + stack = popValue(stack) + } + return stack +} + +// markStacks models the abstract evaluation stack at every code-unit +// offset of code, returning a slice of length len+1 (one extra slot +// for the fall-through past the last instruction). Each entry is either +// a packed stack, stackUninitialized (unreachable), or stackOverflowed. +// +// CPython: Objects/frameobject.c:1308 mark_stacks +func markStacks(code *objects.Code, length int) []int64 { + stacks := make([]int64, length+1) + for i := 1; i <= length; i++ { + stacks[i] = stackUninitialized + } + stacks[0] = stackEmpty + + cacheFor := func(op compile.Opcode) int { return compile.CacheCount(op) } + + todo := true + for todo { + todo = false + // Scan instructions. + for i := 0; i < length; { + nextStack := stacks[i] + opcode := monitor.GetBaseCodeUnit(code, i) + oparg := 0 + // Consume any EXTENDED_ARG prefix run. + for opcode == compile.EXTENDED_ARG { + oparg = (oparg << 8) | int(code.Code[2*i+1]) + i++ + opcode = monitor.GetBaseCodeUnit(code, i) + stacks[i] = nextStack + } + oparg = (oparg << 8) | int(code.Code[2*i+1]) + nextI := i + cacheFor(opcode) + 1 + if nextStack == stackUninitialized { + i = nextI + continue + } + switch opcode { + case compile.POP_JUMP_IF_FALSE, compile.POP_JUMP_IF_TRUE, + compile.POP_JUMP_IF_NONE, compile.POP_JUMP_IF_NOT_NONE: + j := nextI + oparg + nextStack = popValue(nextStack) + stacks[j] = nextStack + stacks[nextI] = nextStack + case compile.SEND: + j := oparg + i + cacheFor(compile.SEND) + 1 + stacks[j] = nextStack + stacks[nextI] = nextStack + case compile.JUMP_FORWARD: + j := oparg + i + 1 + stacks[j] = nextStack + case compile.JUMP_BACKWARD, compile.JUMP_BACKWARD_NO_INTERRUPT: + j := nextI - oparg + if stacks[j] == stackUninitialized && j < i { + todo = true + } + stacks[j] = nextStack + case compile.GET_ITER, compile.GET_AITER: + nextStack = pushValue(popValue(nextStack), kindIterator) + stacks[nextI] = nextStack + case compile.FOR_ITER: + targetStack := pushValue(nextStack, kindObject) + stacks[nextI] = targetStack + j := oparg + 1 + cacheFor(compile.FOR_ITER) + i + stacks[j] = targetStack + case compile.END_ASYNC_FOR: + nextStack = popValue(popValue(nextStack)) + stacks[nextI] = nextStack + case compile.PUSH_EXC_INFO: + nextStack = pushValue(nextStack, kindExcept) + stacks[nextI] = nextStack + case compile.POP_EXCEPT: + nextStack = popValue(nextStack) + stacks[nextI] = nextStack + case compile.RETURN_VALUE: + // End of block. + case compile.RAISE_VARARGS: + // End of block. + case compile.RERAISE: + // End of block. + case compile.PUSH_NULL: + nextStack = pushValue(nextStack, kindNull) + stacks[nextI] = nextStack + case compile.LOAD_GLOBAL: + nextStack = pushValue(nextStack, kindObject) + if oparg&1 != 0 { + nextStack = pushValue(nextStack, kindNull) + } + stacks[nextI] = nextStack + case compile.LOAD_ATTR: + if oparg&1 != 0 { + nextStack = popValue(nextStack) + nextStack = pushValue(nextStack, kindObject) + nextStack = pushValue(nextStack, kindNull) + } + stacks[nextI] = nextStack + case compile.SWAP: + nextStack = stackSwap(nextStack, oparg) + stacks[nextI] = nextStack + case compile.COPY: + nextStack = pushValue(nextStack, peekStack(nextStack, oparg)) + stacks[nextI] = nextStack + default: + delta := compile.OpcodeStackEffectWithJump(int(opcode), int32(oparg), 0) + for delta < 0 { + nextStack = popValue(nextStack) + delta++ + } + for delta > 0 { + nextStack = pushValue(nextStack, kindObject) + delta-- + } + stacks[nextI] = nextStack + } + i = nextI + } + // Scan the exception table: a reachable protected region seeds + // its handler's stack with the unwound level plus the pushed + // Lasti / Except entries. + tab := code.ExceptionTable + pos := 0 + for pos < len(tab) { + if tab[pos]&excEntryStartBit == 0 { + pos++ + continue + } + startOffset, n, ok := readExcVarint(tab, pos) + if !ok { + break + } + pos += n + _, n, ok = readExcVarint(tab, pos) // size, unused here + if !ok { + break + } + pos += n + handler, n, ok := readExcVarint(tab, pos) + if !ok { + break + } + pos += n + depthLasti, n, ok := readExcVarint(tab, pos) + if !ok { + break + } + pos += n + level := depthLasti >> 1 + lasti := depthLasti & 1 + if startOffset < len(stacks) && stacks[startOffset] != stackUninitialized { + if handler < len(stacks) && stacks[handler] == stackUninitialized { + todo = true + targetStack := popToLevel(stacks[startOffset], level) + if lasti != 0 { + targetStack = pushValue(targetStack, kindLasti) + } + targetStack = pushValue(targetStack, kindExcept) + stacks[handler] = targetStack + } + } + } + } + return stacks +} + +// compatibleStack reports whether a jump from fromStack to toStack is +// legal: the source stack must reduce to the target stack with every +// surviving slot kind-compatible. +// +// CPython: Objects/frameobject.c:1522 compatible_stack +func compatibleStack(fromStack, toStack int64) bool { + if fromStack < 0 || toStack < 0 { + return false + } + for fromStack > toStack { + fromStack = popValue(fromStack) + } + for fromStack != 0 { + fromTop := topOfStack(fromStack) + toTop := topOfStack(toStack) + if !compatibleKind(fromTop, toTop) { + return false + } + fromStack = popValue(fromStack) + toStack = popValue(toStack) + } + return toStack == 0 +} + +// explainIncompatibleStack returns the diagnostic for an illegal jump +// target. +// +// CPython: Objects/frameobject.c:1543 explain_incompatible_stack +func explainIncompatibleStack(toStack int64) string { + if toStack == stackOverflowed { + return "stack is too deep to analyze" + } + if toStack == stackUninitialized { + return "can't jump into an exception handler, or code may be unreachable" + } + switch topOfStack(toStack) { + case kindExcept: + return "can't jump into an 'except' block as there's no exception" + case kindLasti: + return "can't jump into a re-raising block as there's no location" + case kindObject, kindNull: + return "incompatible stacks" + case kindIterator: + return "can't jump into the body of a for loop" + default: + return "incompatible stacks" + } +} + +// marklines returns, for each code-unit offset, the source line that +// starts there, or -1. Mirrors CPython walking the line table and +// recording a line only at the first offset where it changes. +// +// CPython: Objects/frameobject.c:1569 marklines +func marklines(code *objects.Code, length int) []int { + linestarts := make([]int, length) + for i := range linestarts { + linestarts[i] = -1 + } + last := -1 + for _, e := range objects.CoLines(code) { + idx := e.Start / 2 + if e.Line != last && e.Line != -1 && idx >= 0 && idx < length { + linestarts[idx] = e.Line + last = e.Line + } + } + return linestarts +} + +// CPython: Objects/frameobject.c:1595 first_line_not_before +func firstLineNotBefore(lines []int, line int) int { + const intMax = int(^uint(0) >> 1) + result := intMax + for _, l := range lines { + if l < result && l >= line { + result = l + } + } + if result == intMax { + return -1 + } + return result +} + +// frameLinenoSetHook backs objects.SetLinenoHook. It extracts the +// activation record from the wrapper and runs frameLinenoSet. +func frameLinenoSetHook(w *objects.Frame, value objects.Object) error { + if w == nil { + return fmt.Errorf("RuntimeError: lost frame") + } + f, ok := w.Interp().(*frame.Frame) + if !ok || f == nil { + return fmt.Errorf("RuntimeError: frame has no activation record") + } + return frameLinenoSet(f, value) +} + +// frameLinenoSet implements the f_lineno setter for a live frame. +// +// CPython: Objects/frameobject.c:1640 frame_lineno_set_impl +// +//nolint:gocognit,gocyclo // mirrors frame_lineno_set_impl: event gate + stack-analysis pop loop +func frameLinenoSet(f *frame.Frame, value objects.Object) error { + ts := currentThread() + code := f.Code + if value == nil { + return fmt.Errorf("ValueError: can't delete numeric/char attribute") + } + iv, ok := value.(*objects.Int) + if !ok { + return fmt.Errorf("ValueError: lineno must be an integer") + } + + // Jumps are forbidden outside a trace callback and from any event + // other than the handful that leave the eval loop re-enterable. + whatEvent := -1 + if ts != nil { + whatEvent = ts.WhatEvent + } + if whatEvent < 0 { + return fmt.Errorf("ValueError: f_lineno can only be set in a trace function") + } + switch monitor.Event(whatEvent) { + case monitor.EventPyResume, monitor.EventJump, monitor.EventBranch, + monitor.EventBranchLeft, monitor.EventBranchRight, + monitor.EventLine, monitor.EventPyYield: + // Allowed. + case monitor.EventPyStart: + return fmt.Errorf("ValueError: can't jump from the 'call' trace event of a new frame") + case monitor.EventCall, monitor.EventCReturn: + return fmt.Errorf("ValueError: can't jump during a call") + case monitor.EventPyReturn, monitor.EventPyUnwind, monitor.EventPyThrow, + monitor.EventRaise, monitor.EventCRaise, monitor.EventInstruction, + monitor.EventExceptionHandled: + return fmt.Errorf("ValueError: can only jump from a 'line' trace event") + default: + return fmt.Errorf("SystemError: unexpected event type") + } + + l, ok := iv.Int64() + const intMax = int64(int32(^uint32(0) >> 1)) + if !ok || l > intMax || l < -intMax-1 { + return fmt.Errorf("ValueError: lineno out of range") + } + newLineno := int(l) + + if newLineno < code.Firstlineno { + return fmt.Errorf("ValueError: line %d comes before the current code block", newLineno) + } + + length := len(code.Code) / 2 + lines := marklines(code, length) + newLineno = firstLineNotBefore(lines, newLineno) + if newLineno < 0 { + return fmt.Errorf("ValueError: line %d comes after the current code block", int(l)) + } + + stacks := markStacks(code, length) + + bestStack := stackOverflowed + bestAddr := -1 + lasti := f.InstrPtr / 2 + startStack := stacks[lasti] + found := false + msg := "cannot find bytecode for specified line" + for i := 0; i < length; i++ { + if lines[i] != newLineno { + continue + } + targetStack := stacks[i] + if compatibleStack(startStack, targetStack) { + found = true + if targetStack > bestStack { + bestStack = targetStack + bestAddr = i + } + } else if !found { + switch { + case startStack == stackOverflowed: + msg = "stack to deep to analyze" + case startStack == stackUninitialized: + msg = "can't jump from unreachable code" + default: + msg = explainIncompatibleStack(targetStack) + } + } + } + if bestAddr < 0 { + return fmt.Errorf("ValueError: %s", msg) + } + + // Bind any locals the compiler proved live at the target but that + // are currently NULL: rather than crash or rewrite co_code, assign + // None (after a RuntimeWarning). + // + // CPython: Objects/frameobject.c:1789 (PyErr_WarnFormat block) + nlocalsplus := frame.NLocalsPlusOf(code) + unbound := 0 + for i := 0; i < nlocalsplus; i++ { + if f.LocalsPlus[i].IsNull() { + unbound++ + } + } + if unbound != 0 { + s := "" + if unbound != 1 { + s = "s" + } + if err := warnings.WarnFormat(errors.PyExc_RuntimeWarning, 0, + "assigning None to %d unbound local%s", unbound, s); err != nil { + return err + } + // Second pass so that warnings-as-errors raises before any + // slot is written. + for i := 0; i < nlocalsplus; i++ { + if f.LocalsPlus[i].IsNull() { + f.LocalsPlus[i] = stackref.FromObjectImmortal(objects.None()) + } + } + } + + if frameIsSuspended(f) { + // Account for the value popped by yield. + startStack = popValue(startStack) + } + + for startStack > bestStack { + popped := f.PopStack() + if topOfStack(startStack) == kindExcept { + obj := popped.AsObjectSteal() + if obj == nil || obj == objects.None() { + ts.SetHandledException(nil) + } else if exc, ok := obj.(state.Exception); ok { + ts.SetHandledException(exc) + } else { + ts.SetHandledException(nil) + } + } else { + popped.Close() + } + startStack = popValue(startStack) + } + + f.Lineno = 0 + f.InstrPtr = bestAddr * 2 + f.LinenoJumped = true + return nil +} + +// frameIsSuspended reports whether f is a generator/coroutine/async-gen +// activation record that is currently suspended (its driver goroutine is +// not running it). A jump in that state must account for the value the +// yield popped. +// +// CPython: Objects/frameobject.c:1610 frame_is_suspended +func frameIsSuspended(f *frame.Frame) bool { + if f.Owner != frame.OwnedByGenerator { + return false + } + switch g := f.GenOwner.(type) { + case *objects.Generator: + return g.Running.Load() == 0 + case *objects.Coroutine: + return g.Running.Load() == 0 + case *objects.AsyncGenerator: + return g.Running.Load() == 0 + } + return false +} diff --git a/vm/legacy_tracing.go b/vm/legacy_tracing.go index 18f9ea369..a5655e503 100644 --- a/vm/legacy_tracing.go +++ b/vm/legacy_tracing.go @@ -121,7 +121,9 @@ func callTraceFunc(event int, arg objects.Object) (objects.Object, error) { } } ts.EnterTracing() + ts.WhatEvent = legacyTraceToEvent(event) err := fn(obj, f, event, arg) + ts.WhatEvent = -1 ts.LeaveTracing() if err != nil { return nil, err @@ -130,6 +132,31 @@ func callTraceFunc(event int, arg objects.Object) (objects.Object, error) { return objects.None(), nil } +// legacyTraceToEvent maps a legacy PyTrace_* event to the PEP 669 +// monitoring event the f_lineno setter switches on. The setter only +// permits a line jump from a 'line' event; the other events resolve to +// the monitoring IDs whose setter arms raise the matching diagnostic +// (PY_START for 'call', PY_RETURN for 'return', and so on). +// +// CPython: Python/legacy_tracing.c the per-event sys_trace_* shims set +// tstate->what_event before invoking the Python callback. +func legacyTraceToEvent(event int) int { + switch event { + case PyTraceLine: + return int(monitor.EventLine) + case PyTraceCall: + return int(monitor.EventPyStart) + case PyTraceReturn: + return int(monitor.EventPyReturn) + case PyTraceException: + return int(monitor.EventRaise) + case PyTraceOpcode: + return int(monitor.EventInstruction) + default: + return int(monitor.EventPyStart) + } +} + // setOpcodeTrace toggles per-instruction tracing on f.Code so the // LINE-mode callback can opt into PyTrace_OPCODE for that frame. // Mirrors set_opcode_trace_world_stopped from CPython, minus the @@ -299,7 +326,9 @@ func sysTraceInstructionFunc(args []objects.Object, _ map[string]objects.Object) return objects.None(), nil } ts.EnterTracing() + ts.WhatEvent = int(monitor.EventInstruction) err := fn(obj, f, PyTraceOpcode, objects.None()) + ts.WhatEvent = -1 ts.LeaveTracing() if err != nil { return nil, err @@ -328,7 +357,9 @@ func traceLine(ts *state.Thread, f *frame.Frame, line int) (objects.Object, erro } f.Lineno = line ts.EnterTracing() + ts.WhatEvent = int(monitor.EventLine) err := fn(obj, f, PyTraceLine, objects.None()) + ts.WhatEvent = -1 ts.LeaveTracing() if err != nil { return nil, err