Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 100 additions & 19 deletions LeanPy/Kernel.lean
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ def allDecls (_ : Unit) : IO String := do
match (← envRef.get) with
| some env =>
let r ← IO.mkRef (#[] : Array String)
env.constants.forM (fun n _ => do r.modify (·.push (toString n)))
env.constants.forM (fun n _ => do r.modify (·.push (n.toString (escape := false))))
let acc ← r.get
return "\n".intercalate acc.qsort.toList
| none => return ""
Expand All @@ -117,7 +117,7 @@ def catalogPy (_ : Unit) : IO String := do
match (← envRef.get) with
| some env =>
let names := envCatalog env
let strs := names.map toString
let strs := names.map (·.toString (escape := false))
return "\n".intercalate strs.qsort.toList
| none => return ""

Expand All @@ -127,30 +127,39 @@ def search (needle : String) : IO String := do
| some env =>
let r ← IO.mkRef (#[] : Array String)
env.constants.forM fun n _ => do
let s := toString n
let s := n.toString (escape := false)
if needle.isEmpty || ((s.splitOn needle).length > 1) then
r.modify (·.push s)
let acc ← r.get
return "\n".intercalate acc.qsort.toList
| none => return ""

/-- A declaration name from Python: escaped (`Foo.«a b»`, as `String.toName`
reads it) or unescaped, as the kernel ops now return names (`Foo.a b`, split
on dots). -/
def parseName (s : String) : Name :=
let n := s.toName
if n.isAnonymous && !s.isEmpty then
(s.splitOn ".").foldl Name.str .anonymous
else n

@[python "leanpy_kernel_decl_exists"]
def declExists (name : String) : IO Bool := do
match (← envRef.get) with
| some env => return (env.find? name.toName).isSome
| some env => return (env.find? (parseName name)).isSome
| none => return false

@[python "leanpy_kernel_decl_type"]
def declType (name : String) : IO String := runCore do
let n := name.toName
let n := parseName name
match (← getEnv).find? n with
| none => return ""
| some info =>
Meta.MetaM.run' do ppExprStr info.type

@[python "leanpy_kernel_decl_value"]
def declValue (name : String) : IO String := runCore do
let n := name.toName
let n := parseName name
match (← getEnv).find? n with
| none => return ""
| some info =>
Expand All @@ -162,23 +171,23 @@ def declValue (name : String) : IO String := runCore do
def moduleOfNameStr (name : String) : IO String := do
match (← envRef.get) with
| some env =>
match module_of_name env name.toName with
| some n => return toString n
match module_of_name env (parseName name) with
| some n => return n.toString (escape := false)
| none => return ""
| none => return ""

@[python "leanpy_kernel_is_internal_name_str"]
def isInternalNameStr (name : String) : IO Bool :=
return isNameInternal name.toName
return isNameInternal (parseName name)

@[python "leanpy_kernel_decl_axioms"]
def declAxioms (name : String) : IO String := runCore do
let env ← getEnv
let n := name.toName
let n := parseName name
if env.find? n |>.isNone then
return ""
let axioms ← collectAxioms n
let names := axioms.map toString |>.qsort
let names := axioms.map (·.toString (escape := false)) |>.qsort
return "\n".intercalate names.toList

/-! ## Elaboration -/
Expand Down Expand Up @@ -660,23 +669,95 @@ def frontendFindSourcePath (moduleName : String) : IO String := do
return p.toString
catch e => return s!"<error: {e.toString}>"

/-- Set up the frontend for `source` against `env`. A header (`module`,
`prelude`, `import …`) is skipped when every module it imports is already in
`env`; otherwise this throws, naming the missing modules. (Before, the header
was parsed as commands, failed, and the error was dropped: empty output.) -/
def frontendContextState (env : Environment) (source : String) (fileName : String) :
IO (Elab.Frontend.Context × Elab.Frontend.State × MessageLog) := do
let inputCtx := Parser.mkInputContext source fileName
let (header, parserState, headerMsgs) ← Parser.parseHeader inputCtx
let hstx : Elab.HeaderSyntax := header
let missing := (hstx.imports (includeInit := false)).filter
fun i => (env.getModuleIdx? i.module).isNone
unless missing.isEmpty do
let ms := missing.toList.map (·.module.toString (escape := false))
throw <| IO.userError s!"the source imports modules not in the loaded environment: {", ".intercalate ms}; load them first (Kernel.load)"
let commandState := Elab.Command.mkState env {} {}
let ctx : Elab.Frontend.Context := { inputCtx }
let st : Elab.Frontend.State := {
commandState := { commandState with infoState.enabled := true },
parserState,
cmdPos := parserState.pos }
return (ctx, st, headerMsgs)

/-- Run `f` on each command of `source` against `env`: its results and the
environment after the last command. -/
def frontendRun (env : Environment) (source : String) (fileName : String)
(f : Pantograph.Frontend.CompilationStep → IO α) :
IO (List α × Environment × MessageLog) := do
let (ctx, st, headerMsgs) ← frontendContextState env source fileName
let m : Pantograph.Frontend.FrontendM (List α) :=
Pantograph.Frontend.mapCompilationSteps fun step => do f step
let (results, st') ← (m.run {} |>.run ctx |>.run st)
return (results, st'.commandState.env, headerMsgs)

/-- Process a single Lean source string against the current environment.
Returns a newline-separated list of constants defined per command. -/
Returns a newline-separated list of constants defined per command (names
unescaped), commands separated by `\n---\n`; `<error: …>` on failure. -/
@[python "leanpy_kernel_frontend_process"]
def frontendProcess (source : String) : IO String := do
match (← envRef.get) with
| none => return "no environment loaded"
| some env =>
try
let (fctx, fstate) ← Pantograph.Frontend.createContextStateFromFile source "<process>" (env? := some env) {}
let m : Pantograph.Frontend.FrontendM (List String) :=
Pantograph.Frontend.mapCompilationSteps fun step => do
let names ← step.newConstants
return "\n".intercalate (names.toList.map toString)
let (steps, _) ← (m.run {} |>.run fctx |>.run fstate)
let (steps, _, _) ← frontendRun env source "<process>" fun step => do
let names ← step.newConstants
return "\n".intercalate (names.toList.map (·.toString (escape := false)))
return "\n---\n".intercalate steps
catch e => return s!"<error: {e.toString}>"

/-- A message as JSON: severity, its range (1-based lines, 0-based columns,
in the processed source) and its text. -/
def messageJson (m : Message) : IO Json := do
let severity := match m.severity with
| .error => "error" | .warning => "warning" | .information => "information"
let endPos := m.endPos.getD m.pos
return Json.mkObj [
("severity", severity), ("line", m.pos.line), ("column", m.pos.column),
("endLine", endPos.line), ("endColumn", endPos.column),
("text", ← m.data.toString)]

/-- Process `source` against the current environment, returning JSON:
`{"error", "header": [messages], "commands": [{"line", "constants",
"messages"}]}`, `error` null unless the source could not be processed (no
environment, an import not loaded); elaboration errors are messages. With `inheritEnv`, the active environment becomes the one
after the source, so later calls (`process`, `decl_type`, `decl_axioms`,
`goal_create`, …) see its declarations; on an exception it is unchanged. -/
@[python "leanpy_kernel_frontend_process_json"]
def frontendProcessJson (source : String) (inheritEnv : Bool) : IO String := do
match (← envRef.get) with
| none =>
return (Json.mkObj [("error", "no environment loaded"),
("header", Json.arr #[]), ("commands", Json.arr #[])]).compress
| some env =>
try
let (cmds, env', headerMsgs) ← frontendRun env source "<process>" fun step => do
let names ← step.newConstants
let msgs ← step.msgs.mapM messageJson
let line := (step.fileMap.toPosition (step.stx.getPos?.getD 0)).line
return Json.mkObj [
("line", line),
("constants", toJson ((names.toList.map (·.toString (escape := false))).toArray.qsort (· < ·))),
("messages", Json.arr msgs.toArray)]
let header ← headerMsgs.toList.mapM messageJson
if inheritEnv then envRef.set (some env')
return (Json.mkObj [("error", Json.null),
("header", Json.arr header.toArray), ("commands", Json.arr cmds.toArray)]).compress
catch e =>
return (Json.mkObj [("error", e.toString),
("header", Json.arr #[]), ("commands", Json.arr #[])]).compress

/-- Collect all `sorry` positions in the source as a draftable goal state.
Returns the goal state pretty-printed (for now). -/
@[python "leanpy_kernel_frontend_collect_sorrys"]
Expand All @@ -685,7 +766,7 @@ def frontendCollectSorrys (source : String) : IO (Option GoalState × String) :=
| none => return (none, "no environment loaded")
| some env =>
try
let (fctx, fstate) ← Pantograph.Frontend.createContextStateFromFile source "<collect_sorrys>" (env? := some env) {}
let (fctx, fstate, _) ← frontendContextState env source "<collect_sorrys>"
let m : Pantograph.Frontend.FrontendM (List Pantograph.Frontend.InfoWithContext) := do
let xss ← Pantograph.Frontend.mapCompilationSteps fun step => Pantograph.Frontend.collectSorrys step
return xss.flatten
Expand Down
21 changes: 21 additions & 0 deletions examples/kernel/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,27 @@ The bridge imports `LeanPy.Kernel` and `LeanPy.Export`. It does not import the
`LeanPy` umbrella or request a Regex archive, although building the existing
LeanPy static library still builds its Regex dependency.

## Elaborating source against the project

`Kernel.elaborate(source, inherit_env=False)` processes Lean commands against
the loaded environment and returns a `ProcessResult`: per command the
constants it added (names unescaped) and Lean's messages (`severity`, `line`,
`column`, `end_line`, `end_column`, `text`), plus `ok`, `errors` and `error`.
A header (`module`, `import …`) is skipped when its imports are loaded, and is
an `error` naming the missing modules otherwise. With `inherit_env=True` the
declarations persist: later `elaborate`, `decl_type`, `decl_axioms` and goal
calls see them, so a translator can elaborate and check one fragment at a time.

```python
r = kernel.elaborate("def x : Nat := 1", inherit_env=True)
assert r.ok, "\n".join(map(str, r.errors))
print(kernel.decl_type("x"))
```

The runtime is the toolchain of the project `LeanLibrary.from_lake` loads,
whatever the current directory; `LEANPY_TOOLCHAIN`, `LEANPY_LEAN_PREFIX` or
`lean_py.utils.set_toolchain` override it.

## Temporary dependency workaround

Regex `v4.34.0-rc1` triggers a Lean 4.35 `grind` panic in
Expand Down
9 changes: 5 additions & 4 deletions lean_py/_parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def find_lean_header() -> Path:
"""Locate lean.h via the active Lean toolchain.

Strategy:
1. Ask `lean --print-prefix` for the toolchain sysroot, look for
1. Ask `utils.lean_prefix` for the toolchain sysroot, look for
`<prefix>/include/lean/lean.h`. This is the canonical location
and works regardless of whether elan, lakefile, or a manual
install picked the toolchain.
Expand All @@ -87,11 +87,12 @@ def find_lean_header() -> Path:
return cand

try:
prefix = subprocess.check_output(["lean", "--print-prefix"], text=True).strip()
header = Path(prefix) / "include" / "lean" / "lean.h"
from lean_py.utils import lean_include_dir # independent of the cwd

header = lean_include_dir() / "lean" / "lean.h"
if header.exists():
return header
except (FileNotFoundError, subprocess.CalledProcessError):
except (FileNotFoundError, RuntimeError):
pass

toolchain_file = Path(__file__).parent.parent / "lean-toolchain"
Expand Down
108 changes: 104 additions & 4 deletions lean_py/kernel.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@

from __future__ import annotations

import json
from collections.abc import Iterable
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Any


Expand Down Expand Up @@ -67,6 +68,87 @@ def parse(cls, encoded: str, kernel: Kernel, raw_state: Any) -> TacticResult:
return cls(status, messages, state)


@dataclass(frozen=True)
class Message:
"""A Lean message: severity (``"error"``, ``"warning"``,
``"information"``), its range in the processed source (1-based lines,
0-based columns) and its text."""

severity: str
line: int
column: int
end_line: int
end_column: int
text: str

@classmethod
def from_json(cls, d: dict) -> Message:
return cls(d["severity"], d["line"], d["column"], d["endLine"], d["endColumn"], d["text"])

def __str__(self) -> str:
return f"{self.line}:{self.column}: {self.severity}: {self.text}"


@dataclass(frozen=True)
class Command:
"""One command of a processed source: the line it starts on, the
constants it added (names unescaped) and its messages."""

line: int
constants: list[str]
messages: list[Message]


@dataclass(frozen=True)
class ProcessResult:
"""The result of :meth:`Kernel.elaborate`.

Attributes:
commands: each command's constants and messages, in order.
header: messages from parsing the source's header, if any.
error: why the source could not be processed at all (no environment
loaded, an imported module not loaded), else ``None``.
"""

commands: list[Command] = field(default_factory=list)
header: list[Message] = field(default_factory=list)
error: str | None = None

@property
def messages(self) -> list[Message]:
"""Every message, the header's first."""
return [*self.header, *(m for c in self.commands for m in c.messages)]

@property
def errors(self) -> list[Message]:
"""The messages of severity ``"error"``."""
return [m for m in self.messages if m.severity == "error"]

@property
def constants(self) -> list[str]:
"""Every constant the source added, in command order."""
return [n for c in self.commands for n in c.constants]

@property
def ok(self) -> bool:
"""Processed, with no error message."""
return self.error is None and not self.errors

@classmethod
def from_json(cls, encoded: str) -> ProcessResult:
d = json.loads(encoded)
return cls(
[
Command(
c["line"], list(c["constants"]), [Message.from_json(m) for m in c["messages"]]
)
for c in d["commands"]
],
[Message.from_json(m) for m in d["header"]],
d["error"],
)


class GoalState:
"""Opaque handle to a Lean ``GoalState``. Methods dispatch back into the
underlying Lean library."""
Expand Down Expand Up @@ -381,10 +463,28 @@ def find_source_path(self, module_name: str) -> str:
def process(self, source: str) -> str:
"""Process Lean source code against the current environment.

Returns a multi-line string of new constants per command, separated by
``\\n---\\n``."""
Returns a multi-line string of new constants per command (names
unescaped), separated by ``\\n---\\n``, or ``<error: …>``. A header
(``module``, ``import …``) is skipped when its imports are loaded and
is an error otherwise. Messages are dropped: see :meth:`elaborate`."""
return self._lib.leanpy_kernel_frontend_process(source)

def elaborate(self, source: str, *, inherit_env: bool = False) -> ProcessResult:
"""Process Lean source code against the current environment, keeping
Lean's messages.

A header (``module``, ``prelude``, ``import …``) is skipped when every
module it imports is already loaded; otherwise ``error`` names the
missing modules and nothing is elaborated. With ``inherit_env=True``
the active environment becomes the one after the source, so later
calls (``elaborate``, ``decl_type``, ``decl_axioms``, ``goal_create``,
…) see its declarations. A declaration that fails to elaborate is
still added (with ``sorryAx``) and reported; check ``ok`` / ``errors``.
"""
return ProcessResult.from_json(
self._lib.leanpy_kernel_frontend_process_json(source, inherit_env)
)

def collect_sorrys(self, source: str) -> tuple[GoalState | None, str]:
"""Extract all `sorry` placeholders in ``source`` as a draftable
:class:`GoalState`. Returns ``(state, message)`` — state is ``None`` if
Expand All @@ -407,4 +507,4 @@ def expr_proj_to_app(self, src: str) -> str:
return self._lib.leanpy_kernel_delab_expr_proj_to_app(src)


__all__ = ["Kernel", "GoalState", "TacticResult"]
__all__ = ["Command", "GoalState", "Kernel", "Message", "ProcessResult", "TacticResult"]
Loading
Loading