diff --git a/README.md b/README.md index 78c5715..dddfeea 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,32 @@ trueup.reconcile(Table.file("statement.csv"), Table.file("receiving.csv"), new R Each call to `reconcile` or `reconcileFiles` counts as one analysis on your plan. +## Stored files, runs and saved models + +Files uploaded to your team stay there (you'll also see them in the dashboard). Runs on stored files are kept, and what a run learned can be saved as a model: + +```java +List files = trueup.uploadFiles(Table.file("statement.csv"), Table.file("receiving.csv")); +StoredFile statement = files.get(0), receiving = files.get(1); // .rows, .columns, .roles ("Qty" -> "number", ...) + +ReconcileResult result = trueup.reconcileStored(statement.id, receiving.id); +String modelId = trueup.createModel(result.run_id, "Acme statements"); + +// Next month: apply what was learned. +trueup.reconcileStored(List.of(aprilStatement.id, aprilReceiving.id), modelId, null); +``` + +| Method | Returns | +|---|---| +| `uploadFiles(Table...)`, `listFiles()`, `getFile(id)` | `StoredFile`: `id`, `name`, `rows`, `columns`, `roles` | +| `fileContent(id)` | the bytes, exactly as uploaded | +| `deleteFile(id)` | | +| `reconcileStored(leftId, rightId)`, `reconcileStored(fileIds, model, answers)` | a result with `run_id` (one analysis) | +| `listRuns(limit, before)` | `RunPage`: `runs`, `has_more`, newest first | +| `forEachRun(action)` | every run, paging for you | +| `getRun(id)` | `RunDetail`: `run`, `result` | +| `createModel(runId, name)`, `listModels()`, `getModel(id)`, `deleteModel(id)` | `getModel` includes the `weights` | + ## Findings | `kind` | Meaning | @@ -121,7 +147,7 @@ TrueUp.builder() The tests run in Docker against the live API: ```bash -export TRUEUP_API_KEY=tu_live_... # a key for a test team (each run uses 2 analyses) +export TRUEUP_API_KEY=tu_live_... # a key for a test team (each run uses 4 analyses) just test # or: docker compose run --rm test ``` diff --git a/src/main/java/io/github/merchantprotocol/trueup/Models.java b/src/main/java/io/github/merchantprotocol/trueup/Models.java index c1c5b75..363b034 100644 --- a/src/main/java/io/github/merchantprotocol/trueup/Models.java +++ b/src/main/java/io/github/merchantprotocol/trueup/Models.java @@ -49,6 +49,65 @@ public static final class ReconcileResult { public Details details; public List inputs; public String engine; + /** The kept run, for {@link TrueUp#reconcileStored}; null for tables sent inline. */ + public String run_id; + } + + /** A file stored in the team (uploaded through the API or the dashboard). */ + public static final class StoredFile { + public String id; + public String name; + public long size; + /** "table" or "document". */ + public String kind; + public Integer rows; + public List columns; + /** What TrueUp read each column as: "date", "number", "text", ... */ + public Map roles; + public String created_at; + } + + /** A run on stored files, from the API or the dashboard. */ + public static final class Run { + public String id; + public String analysis; + /** "done" or "failed". */ + public String status; + /** "api" or "portal". */ + public String via; + public List inputs; + public Named model; + public String headline; + public Map stats; + /** How many findings the run has. */ + public Integer findings; + public String error; + public String created_at; + + public static final class Named { public String id; public String name; } + } + + /** One page of runs, newest first. */ + public static final class RunPage { + public List runs; + public boolean has_more; + } + + /** One run and its full result (null if the run failed). */ + public static final class RunDetail { + public Run run; + public ReconcileResult result; + } + + /** A saved model: what a run learned, reusable on next month's files. */ + public static final class Model { + public String id; + public String name; + public String analysis; + public String source_run_id; + public String created_at; + /** Only from {@link TrueUp#getModel}. */ + public JsonObject weights; } /** The team, plan and key behind an API key. */ diff --git a/src/main/java/io/github/merchantprotocol/trueup/TrueUp.java b/src/main/java/io/github/merchantprotocol/trueup/TrueUp.java index 409e545..add62a7 100644 --- a/src/main/java/io/github/merchantprotocol/trueup/TrueUp.java +++ b/src/main/java/io/github/merchantprotocol/trueup/TrueUp.java @@ -6,6 +6,11 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import io.github.merchantprotocol.trueup.Models.Account; +import io.github.merchantprotocol.trueup.Models.Answers; +import io.github.merchantprotocol.trueup.Models.Model; +import io.github.merchantprotocol.trueup.Models.RunDetail; +import io.github.merchantprotocol.trueup.Models.RunPage; +import io.github.merchantprotocol.trueup.Models.StoredFile; import io.github.merchantprotocol.trueup.Models.Plan; import io.github.merchantprotocol.trueup.Models.ReconcileOptions; import io.github.merchantprotocol.trueup.Models.ReconcileResult; @@ -13,6 +18,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URI; +import java.net.URLEncoder; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; @@ -23,6 +29,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.function.Consumer; /** * TrueUp API client. Thread-safe. @@ -142,9 +149,122 @@ public ReconcileResult reconcileFiles(List files, ReconcileOptions option return upload(fields, files, options != null ? options : new ReconcileOptions()); } + // ---------------------------------------------------------------- stored files, runs, saved models + + /** Upload one or more files to the team. Each comes back with its id, rows, columns and roles. */ + public List uploadFiles(Table... files) { + if (files.length == 0) throw new TrueUpException.InvalidRequestException("Pass at least one file to upload.", 0, "invalid_request", null); + List fields = new ArrayList<>(); + for (int i = 0; i < files.length; i++) fields.add("file"); + String json = new String(multipart("/v1/files", fields, List.of(files), new ReconcileOptions()), StandardCharsets.UTF_8); + return list(json, "files", StoredFile.class); + } + + /** The team's stored files. */ + public List listFiles() { + return list(request("GET", "/v1/files", null, null), "files", StoredFile.class); + } + + public StoredFile getFile(String id) { + return gson.fromJson(obj(request("GET", "/v1/files/" + enc(id), null, null)).get("file"), StoredFile.class); + } + + /** The file's bytes, exactly as uploaded. */ + public byte[] fileContent(String id) { + return send("GET", "/v1/files/" + enc(id) + "/content", null, null); + } + + public void deleteFile(String id) { + request("DELETE", "/v1/files/" + enc(id), null, null); + } + + /** + * Reconcile two files already stored in the team, by id: {@code leftFileId} bills or claims. The run is kept: + * its id is {@code run_id} in the result. One analysis. + */ + public ReconcileResult reconcileStored(String leftFileId, String rightFileId) { + return reconcileStored(Map.of("left_file_id", leftFileId, "right_file_id", rightFileId), null, null); + } + + /** Reconcile stored files by id; TrueUp picks the pair. {@code model} (a saved model id) and {@code answers} may be null. */ + public ReconcileResult reconcileStored(List fileIds, String model, Answers answers) { + return reconcileStored(Map.of("file_ids", fileIds), model, answers); + } + + private ReconcileResult reconcileStored(Map ids, String model, Answers answers) { + Map body = new LinkedHashMap<>(ids); + if (model != null) body.put("model", model); + if (answers != null) body.put("answers", answers); + return gson.fromJson(request("POST", "/v1/reconcile", gson.toJson(body).getBytes(StandardCharsets.UTF_8), "application/json"), + ReconcileResult.class); + } + + /** One page of runs on stored files, newest first. {@code limit} 1-100 (null for 100); {@code before} a run id or null. */ + public RunPage listRuns(Integer limit, String before) { + List q = new ArrayList<>(); + if (limit != null) q.add("limit=" + limit); + if (before != null) q.add("before=" + enc(before)); + return gson.fromJson(request("GET", "/v1/runs" + (q.isEmpty() ? "" : "?" + String.join("&", q)), null, null), RunPage.class); + } + + /** Every run, newest first, fetching page after page. */ + public void forEachRun(Consumer action) { + String before = null; + while (true) { + RunPage page = listRuns(100, before); + page.runs.forEach(action); + if (!page.has_more || page.runs.isEmpty()) return; + before = page.runs.get(page.runs.size() - 1).id; + } + } + + /** One run and its full result, in the shape {@link #reconcile} returns. */ + public RunDetail getRun(String id) { + return gson.fromJson(request("GET", "/v1/runs/" + enc(id), null, null), RunDetail.class); + } + + /** Save what a run learned as a model. Returns the model id. {@code name} may be null. */ + public String createModel(String runId, String name) { + Map body = new LinkedHashMap<>(); + body.put("run_id", runId); + if (name != null) body.put("name", name); + return obj(request("POST", "/v1/models", gson.toJson(body).getBytes(StandardCharsets.UTF_8), "application/json")).get("id").getAsString(); + } + + public List listModels() { + return list(request("GET", "/v1/models", null, null), "models", Model.class); + } + + /** One saved model, including its weights. */ + public Model getModel(String id) { + return gson.fromJson(obj(request("GET", "/v1/models/" + enc(id), null, null)).get("model"), Model.class); + } + + public void deleteModel(String id) { + request("DELETE", "/v1/models/" + enc(id), null, null); + } + + private static String enc(String s) { + return URLEncoder.encode(s, StandardCharsets.UTF_8).replace("+", "%20"); + } + + private static JsonObject obj(String json) { + return JsonParser.parseString(json).getAsJsonObject(); + } + + private List list(String json, String key, Class type) { + List out = new ArrayList<>(); + for (JsonElement e : obj(json).getAsJsonArray(key)) out.add(gson.fromJson(e, type)); + return out; + } + // ---------------------------------------------------------------- transport private ReconcileResult upload(List fields, List
tables, ReconcileOptions o) { + return gson.fromJson(new String(multipart("/v1/reconcile", fields, tables, o), StandardCharsets.UTF_8), ReconcileResult.class); + } + + private byte[] multipart(String path, List fields, List
tables, ReconcileOptions o) { String boundary = "----trueup" + Long.toHexString(RANDOM.nextLong()) + Long.toHexString(RANDOM.nextLong()); ByteArrayOutputStream body = new ByteArrayOutputStream(); try { @@ -162,8 +282,7 @@ private ReconcileResult upload(List fields, List
tables, Reconcil } catch (IOException e) { throw new TrueUpException.ConnectionException(e.getMessage()); } - String json = request("POST", "/v1/reconcile", body.toByteArray(), "multipart/form-data; boundary=" + boundary); - return gson.fromJson(json, ReconcileResult.class); + return send("POST", path, body.toByteArray(), "multipart/form-data; boundary=" + boundary); } private static void field(ByteArrayOutputStream out, String boundary, String name, String value) throws IOException { @@ -172,6 +291,10 @@ private static void field(ByteArrayOutputStream out, String boundary, String nam } private String request(String method, String path, byte[] body, String contentType) { + return new String(send(method, path, body, contentType), StandardCharsets.UTF_8); + } + + private byte[] send(String method, String path, byte[] body, String contentType) { for (int attempt = 0; ; attempt++) { HttpRequest.Builder req = HttpRequest.newBuilder(URI.create(baseUrl + path)) .timeout(timeout) @@ -183,9 +306,9 @@ private String request(String method, String path, byte[] body, String contentTy } else { req.method(method, HttpRequest.BodyPublishers.noBody()); } - HttpResponse res; + HttpResponse res; try { - res = http.send(req.build(), HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); + res = http.send(req.build(), HttpResponse.BodyHandlers.ofByteArray()); } catch (IOException e) { if (attempt < maxRetries) { sleep(backoff(attempt)); @@ -198,10 +321,11 @@ private String request(String method, String path, byte[] body, String contentTy } int status = res.statusCode(); if (status >= 200 && status < 300) return res.body(); + String text = new String(res.body(), StandardCharsets.UTF_8); String code = "http_" + status; String message = "HTTP " + status; try { - JsonObject err = JsonParser.parseString(res.body()).getAsJsonObject().getAsJsonObject("error"); + JsonObject err = JsonParser.parseString(text).getAsJsonObject().getAsJsonObject("error"); if (err != null) { if (err.has("code")) code = err.get("code").getAsString(); if (err.has("message")) message = err.get("message").getAsString(); @@ -209,7 +333,7 @@ private String request(String method, String path, byte[] body, String contentTy } catch (RuntimeException ignored) { // not a JSON error body } - TrueUpException error = errorFor(status, code, message, res.body(), res.headers().firstValue("retry-after").orElse(null)); + TrueUpException error = errorFor(status, code, message, text, res.headers().firstValue("retry-after").orElse(null)); boolean retryable = error instanceof TrueUpException.RateLimitException || error instanceof TrueUpException.ServerException; if (retryable && attempt < maxRetries) { Double ra = error instanceof TrueUpException.RateLimitException ? ((TrueUpException.RateLimitException) error).getRetryAfter() : null; diff --git a/src/test/java/io/github/merchantprotocol/trueup/ApiTest.java b/src/test/java/io/github/merchantprotocol/trueup/ApiTest.java index ae8a49d..ba4e23c 100644 --- a/src/test/java/io/github/merchantprotocol/trueup/ApiTest.java +++ b/src/test/java/io/github/merchantprotocol/trueup/ApiTest.java @@ -1,12 +1,17 @@ package io.github.merchantprotocol.trueup; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; import io.github.merchantprotocol.trueup.Models.ReconcileOptions; +import io.github.merchantprotocol.trueup.Models.RunDetail; +import io.github.merchantprotocol.trueup.Models.RunPage; +import io.github.merchantprotocol.trueup.Models.StoredFile; import io.github.merchantprotocol.trueup.Models.ReconcileResult; import java.io.IOException; import java.nio.file.Files; @@ -19,7 +24,7 @@ /** * Integration tests against the live TrueUp API. Need TRUEUP_API_KEY (and optionally TRUEUP_BASE_URL). - * Each full run uses 2 analyses. Run in Docker: `just test` (or `docker compose run --rm test`). + * Each full run uses 4 analyses. Run in Docker: `just test` (or `docker compose run --rm test`). */ class ApiTest { private static final Path FIXTURES = Path.of("src/test/resources/fixtures"); @@ -90,4 +95,45 @@ void errorsAreTyped() { assertEquals(422, bad.getStatus()); assertEquals("unsupported_file", bad.getCode()); } + + @Test + void storedFilesRunsAndModels() throws IOException { + assumeTrue(live(), "needs TRUEUP_API_KEY"); + TrueUp tu = TrueUp.builder().build(); + List files = tu.uploadFiles(Table.file(FIXTURES.resolve("statement.csv")), Table.file(FIXTURES.resolve("receiving.csv"))); + StoredFile statement = files.get(0); + StoredFile receiving = files.get(1); + try { + assertEquals(8, statement.rows); + assertEquals("number", statement.roles.get("Qty")); + assertEquals("receiving.csv", tu.getFile(receiving.id).name); + assertTrue(tu.listFiles().stream().anyMatch(f -> f.id.equals(statement.id))); + assertArrayEquals(Files.readAllBytes(FIXTURES.resolve("statement.csv")), tu.fileContent(statement.id)); + + ReconcileResult result = tu.reconcileStored(statement.id, receiving.id); + assertEquals(7, result.stats.get("paired")); + assertTrue(result.run_id.startsWith("run_")); + RunDetail run = tu.getRun(result.run_id); + assertEquals("done", run.run.status); + assertEquals(7, run.result.stats.get("paired")); + RunPage page = tu.listRuns(1, null); + assertEquals(1, page.runs.size()); + assertTrue(page.has_more); + assertNotEquals(page.runs.get(0).id, tu.listRuns(1, page.runs.get(0).id).runs.get(0).id); + + String modelId = tu.createModel(result.run_id, "sdk test"); + try { + assertEquals("trueup.match-weights", tu.getModel(modelId).weights.get("format").getAsString()); + ReconcileResult again = tu.reconcileStored(List.of(statement.id, receiving.id), modelId, null); + assertFalse(again.details.model.get("learned").getAsBoolean()); + } finally { + tu.deleteModel(modelId); + } + assertThrows(TrueUpException.NotFoundException.class, () -> tu.getModel(modelId)); + } finally { + tu.deleteFile(statement.id); + tu.deleteFile(receiving.id); + } + assertThrows(TrueUpException.NotFoundException.class, () -> tu.getFile(statement.id)); + } }