diff --git a/common/src/main/java/com/skyflow/BaseVaultClient.java b/common/src/main/java/com/skyflow/BaseVaultClient.java index 8fb94454..e31687b6 100644 --- a/common/src/main/java/com/skyflow/BaseVaultClient.java +++ b/common/src/main/java/com/skyflow/BaseVaultClient.java @@ -9,6 +9,7 @@ import com.skyflow.generated.common.rest.core.ObjectMappers; import com.skyflow.logs.ErrorLogs; import com.skyflow.logs.InfoLogs; +import com.skyflow.logs.WarningLogs; import com.skyflow.serviceaccount.util.Token; import com.skyflow.utils.BaseConstants; import com.skyflow.utils.BaseUtils; @@ -17,9 +18,12 @@ import io.github.cdimascio.dotenv.Dotenv; import io.github.cdimascio.dotenv.DotenvException; import okhttp3.ConnectionPool; +import okhttp3.Interceptor; import okhttp3.OkHttpClient; import okhttp3.Request; +import okhttp3.Response; +import java.net.HttpURLConnection; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -33,6 +37,17 @@ public class BaseVaultClient { protected BaseCredentials finalCredentials; protected String token; protected String apiKey; + // The token whose regeneration after a 401 last failed. Guarded by this. Further 401s on that + // same token return without generating again, so an auth outage costs one generation per + // token instead of one per rejected request. Cleared whenever a token is stored. + private String refreshFailedFor; + + /** + * A 401 is treated as token expiry only if the token the SDK sent has an {@code exp} at or + * before now plus this window. By the time the 401 arrives the gateway has already judged the + * token expired, so the window only has to absorb clock skew between this host and the gateway. + */ + protected static final long BEARER_TOKEN_EXPIRY_WINDOW_SECONDS = 60; protected BaseVaultClient(V vaultConfig, BaseCredentials credentials) { this.vaultConfig = vaultConfig; @@ -43,18 +58,116 @@ protected V getVaultConfig() { return vaultConfig; } + /** + * The bearer token to send. Synchronized on the same monitor as {@link #setBearerToken(BaseCredentials)} + * and {@link #refreshBearerTokenAfterUnauthorized(String)}, so a request attempt on an OkHttp or + * batch thread sees the token those methods wrote, and waits for a regeneration in progress + * instead of sending the token it is replacing. + */ + protected synchronized String getToken() { + return token; + } + protected OkHttpClient buildSharedHttpClient(Supplier tokenSupplier) { return new OkHttpClient.Builder() .connectionPool(new ConnectionPool(10, 1, TimeUnit.MINUTES)) - .addInterceptor(chain -> { - Request requestWithAuth = chain.request().newBuilder() - .header("Authorization", "Bearer " + tokenSupplier.get()) - .build(); - return chain.proceed(requestWithAuth); - }) + .addInterceptor(authInterceptor(tokenSupplier)) .build(); } + /** + * The auth interceptor shared by every OkHttp client in the SDK. It attaches the current bearer + * token to each attempt, reading it fresh from {@code tokenSupplier} so a token refreshed between + * attempts is picked up. When the vault answers {@code 401} and + * {@link #refreshBearerTokenAfterUnauthorized(String)} can supply a replacement, the rejected + * response is closed and the same request is sent once more with the new token. Any other + * status, and a {@code 401} the refresh declines, is returned as is. + */ + protected Interceptor authInterceptor(Supplier tokenSupplier) { + return chain -> { + String sent = tokenSupplier.get(); + Response response = chain.proceed(withBearer(chain.request(), sent)); + if (response.code() != HttpURLConnection.HTTP_UNAUTHORIZED) { + return response; + } + String fresh = refreshBearerTokenAfterUnauthorized(sent); + if (fresh == null) { + return response; + } + response.close(); + return chain.proceed(withBearer(chain.request(), fresh)); + }; + } + + private static Request withBearer(Request request, String bearer) { + return request.newBuilder().header("Authorization", "Bearer " + bearer).build(); + } + + /** + * Called when the vault rejected {@code rejected} with {@code 401}. Returns the token to resend + * the request with, or {@code null} when the request must fail as it stands: + *
    + *
  • the effective credentials are a static token or an API key, which the SDK cannot replace;
    • + *
  • the rejected token is not at or past expiry (within {@link #BEARER_TOKEN_EXPIRY_WINDOW_SECONDS}), + * so the {@code 401} was not caused by expiry and a new token would not help;
    • + *
  • generating a replacement failed, or produced the very token that was rejected;
    • + *
  • a replacement for this same token was already attempted and failed, so the outcome is + * shared with every request still holding it. A later method call recovers through + * {@link #setBearerToken(BaseCredentials)}, which clears that record when it stores a token.
    • + *
+ * Synchronized with {@link #setBearerToken(BaseCredentials)} and {@link #getToken()} so concurrent + * {@code 401}s on one token share a single regeneration: a caller that finds the cached token + * already different from the one it sent reuses it without generating again. + */ + protected synchronized String refreshBearerTokenAfterUnauthorized(String rejected) { + BaseCredentials credentials = this.finalCredentials; + if (credentials == null || credentials.getApiKey() != null || credentials.getToken() != null) { + return null; + } + if (!Token.isExpiringWithin(rejected, BEARER_TOKEN_EXPIRY_WINDOW_SECONDS)) { + LogUtil.printInfoLog(InfoLogs.BEARER_TOKEN_REJECTED_NOT_DUE.getLog()); + return null; + } + String cached = this.token; + if (cached != null && !cached.equals(rejected)) { + LogUtil.printInfoLog(InfoLogs.BEARER_TOKEN_REJECTED_REUSING_REGENERATED.getLog()); + return cached; + } + // rejected is non-null here: a null token fails the expiry gate above. + if (rejected.equals(refreshFailedFor)) { + LogUtil.printInfoLog(InfoLogs.BEARER_TOKEN_REJECTED_REGENERATION_ALREADY_FAILED.getLog()); + return null; + } + LogUtil.printInfoLog(InfoLogs.BEARER_TOKEN_REJECTED_REGENERATING.getLog()); + try { + String fresh = generateBearerToken(credentials); + if (fresh == null || fresh.equals(rejected)) { + LogUtil.printWarningLog(WarningLogs.BEARER_TOKEN_REGENERATION_RETURNED_REJECTED_TOKEN.getLog()); + refreshFailedFor = rejected; + return null; + } + storeToken(fresh); + return fresh; + } catch (SkyflowException e) { + LogUtil.printWarningLog(WarningLogs.BEARER_TOKEN_REGENERATION_AFTER_401_FAILED.getLog() + + " httpCode=" + e.getHttpCode() + + (e.getRequestId() != null ? " requestId=" + e.getRequestId() : "")); + refreshFailedFor = rejected; + return null; + } + } + + /** Stores a token the SDK obtained and forgets any failed regeneration, so refresh can run again. */ + private void storeToken(String fresh) { + this.token = fresh; + this.refreshFailedFor = null; + } + + /** Single point that turns credentials into a bearer token; tests substitute it. */ + protected String generateBearerToken(BaseCredentials credentials) throws SkyflowException { + return BaseUtils.generateBearerToken(credentials); + } + protected synchronized void prioritiseCredentials(BaseCredentials vaultSpecificCredentials) throws SkyflowException { try { BaseCredentials original = this.finalCredentials; @@ -78,6 +191,7 @@ protected synchronized void prioritiseCredentials(BaseCredentials vaultSpecificC if (original != null && !original.equals(this.finalCredentials)) { token = null; apiKey = null; + refreshFailedFor = null; } } catch (DotenvException e) { throw new SkyflowException(ErrorCode.INVALID_INPUT.getCode(), BaseErrorMessage.EmptyCredentials.getMessage()); @@ -95,10 +209,10 @@ protected synchronized void setBearerToken(BaseCredentials vaultSpecificCredenti LogUtil.printInfoLog(InfoLogs.USE_API_KEY.getLog()); token = this.finalCredentials.getApiKey(); } else if (token == null || token.trim().isEmpty()) { - token = BaseUtils.generateBearerToken(this.finalCredentials); + storeToken(generateBearerToken(this.finalCredentials)); } else if (Token.isExpired(token)) { LogUtil.printInfoLog(InfoLogs.BEARER_TOKEN_EXPIRED.getLog()); - token = BaseUtils.generateBearerToken(this.finalCredentials); + storeToken(generateBearerToken(this.finalCredentials)); } else { LogUtil.printInfoLog(InfoLogs.REUSE_BEARER_TOKEN.getLog()); } diff --git a/common/src/main/java/com/skyflow/detect/DataSourceType.java b/common/src/main/java/com/skyflow/detect/DataSourceType.java deleted file mode 100644 index d152fbb4..00000000 --- a/common/src/main/java/com/skyflow/detect/DataSourceType.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.skyflow.detect; - -/** - * How file content is supplied to, or returned by, Detect file operations. - */ -public enum DataSourceType { - BASE64, - SKYFLOW_ID, - PRESIGNED_URL; -} diff --git a/common/src/main/java/com/skyflow/detect/DeidentifyFileRequest.java b/common/src/main/java/com/skyflow/detect/DeidentifyFileRequest.java index e1d9e6fd..914989dc 100644 --- a/common/src/main/java/com/skyflow/detect/DeidentifyFileRequest.java +++ b/common/src/main/java/com/skyflow/detect/DeidentifyFileRequest.java @@ -3,10 +3,10 @@ import java.io.File; /** - * Request for {@code deidentifyFile}. Supply the file through exactly one of {@code value} (with a - * {@code dataSource}), {@code file} or {@code filePath}. The two file forms are builder sugar that set - * {@code dataSource = BASE64}, encode the content into {@code value} when the request is sent, and infer - * {@code dataFormat} from the extension unless it is set explicitly. + * Request for {@code deidentifyFile}. Supply the file through exactly one of {@code filePath}, {@code file} + * or {@code value} (with a {@code dataSource}); more than one, or none, is rejected before any call. A local + * file given as {@code file} or {@code filePath} is read when the request is sent, sent as {@code BASE64}, and + * its {@code dataFormat} is inferred from the extension unless set explicitly. * *

Without {@code pollOptions} the call submits the run and returns its id. With {@code pollOptions} the * SDK polls the run and, on success for a file-based request, writes the outputs to disk. @@ -16,9 +16,10 @@ * the vault's defaults. */ public final class DeidentifyFileRequest { - private final DataSourceType dataSource; + private final DeidentifyFileSource dataSource; private final String value; private final File file; + private final String filePath; private final FileDataFormat dataFormat; private final String configurationId; private final DetectConfiguration configuration; @@ -29,6 +30,7 @@ private DeidentifyFileRequest(DeidentifyFileRequestBuilder builder) { this.dataSource = builder.dataSource; this.value = builder.value; this.file = builder.file; + this.filePath = builder.filePath; this.dataFormat = builder.dataFormat; this.configurationId = builder.configurationId; this.configuration = builder.configuration; @@ -44,26 +46,34 @@ public static DeidentifyFileRequestBuilder builder() { } /** - * @return How {@code value} supplies the file. Required unless {@code file} or {@code filePath} was used. + * @return How {@code value} supplies the file: this SDK's {@code DeidentifySourceType} (skyvault: BASE64, SKYFLOW_ID, + * PRESIGNED_URL; flowvault: BASE64, PRESIGNED_URL). Required with {@code value}. */ - public DataSourceType getDataSource() { + public DeidentifyFileSource getDataSource() { return dataSource; } /** - * @return Base64 content, skyflow id or presigned URL, according to {@code dataSource}. + * @return The file as {@code dataSource} describes it: base64 content, a presigned URL or, on skyvault, a skyflow id. */ public String getValue() { return value; } /** - * @return Local file to de-identify, when {@code file} or {@code filePath} was used. + * @return Local file to de-identify, when {@code file} was used. */ public File getFile() { return file; } + /** + * @return Path of the local file to de-identify, when {@code filePath} was used. + */ + public String getFilePath() { + return filePath; + } + /** * @return Format of the file. Optional; inferred from the file extension or by the server when omitted. */ @@ -105,6 +115,7 @@ public String toString() { "dataSource=" + dataSource + ", value=" + (value == null ? null : "<" + value.length() + " chars>") + ", file=" + file + + ", filePath=" + filePath + ", dataFormat=" + dataFormat + ", configurationId=" + configurationId + ", configuration=" + configuration + @@ -114,9 +125,10 @@ public String toString() { } public static final class DeidentifyFileRequestBuilder { - private DataSourceType dataSource; + private DeidentifyFileSource dataSource; private String value; private File file; + private String filePath; private FileDataFormat dataFormat; private String configurationId; private DetectConfiguration configuration; @@ -127,16 +139,17 @@ private DeidentifyFileRequestBuilder() { } /** - * @param dataSource How {@code value} supplies the file. + * @param dataSource How {@code value} supplies the file: this SDK's {@code DeidentifySourceType} + * (skyvault: BASE64, SKYFLOW_ID, PRESIGNED_URL; flowvault: BASE64, PRESIGNED_URL). * @return this builder */ - public DeidentifyFileRequestBuilder dataSource(DataSourceType dataSource) { + public DeidentifyFileRequestBuilder dataSource(DeidentifyFileSource dataSource) { this.dataSource = dataSource; return this; } /** - * @param value Base64 content, skyflow id or presigned URL, according to {@code dataSource}. + * @param value The file as {@code dataSource} describes it: base64 content, a presigned URL or, on skyvault, a skyflow id. * @return this builder */ public DeidentifyFileRequestBuilder value(String value) { @@ -145,23 +158,21 @@ public DeidentifyFileRequestBuilder value(String value) { } /** - * @param file Local file to de-identify. Sets {@code dataSource} to BASE64; the content is encoded when sent. + * @param file Local file to de-identify. Sent as BASE64; the content is encoded when sent. * @return this builder */ public DeidentifyFileRequestBuilder file(File file) { this.file = file; - if (file != null) { - this.dataSource = DataSourceType.BASE64; - } return this; } /** - * @param filePath Path of a local file to de-identify. Same as {@link #file(File)}. + * @param filePath Path of a local file to de-identify. Sent as BASE64; the content is encoded when sent. * @return this builder */ public DeidentifyFileRequestBuilder filePath(String filePath) { - return file(filePath == null ? null : new File(filePath)); + this.filePath = filePath; + return this; } /** diff --git a/common/src/main/java/com/skyflow/detect/DeidentifyFileResponse.java b/common/src/main/java/com/skyflow/detect/DeidentifyFileResponse.java index 482b3b63..730177cc 100644 --- a/common/src/main/java/com/skyflow/detect/DeidentifyFileResponse.java +++ b/common/src/main/java/com/skyflow/detect/DeidentifyFileResponse.java @@ -11,12 +11,12 @@ public final class DeidentifyFileResponse { private final String runId; private final DetectRunStatus status; - private final DataSourceType outputType; + private final DeidentifyFileSource outputType; private final List output; private final Metrics metrics; private final String message; - public DeidentifyFileResponse(String runId, DetectRunStatus status, DataSourceType outputType, + public DeidentifyFileResponse(String runId, DetectRunStatus status, DeidentifyFileSource outputType, List output, Metrics metrics, String message) { this.runId = runId; this.status = status; @@ -42,9 +42,9 @@ public DetectRunStatus getStatus() { } /** - * @return How output files are delivered. Null unless polled to SUCCESS. + * @return How output files are delivered. Null unless polled to SUCCESS. A constant of this SDK's {@code DeidentifySourceType}: {@code BASE64} or {@code PRESIGNED_URL}. */ - public DataSourceType getOutputType() { + public DeidentifyFileSource getOutputType() { return outputType; } diff --git a/common/src/main/java/com/skyflow/detect/DeidentifyFileSource.java b/common/src/main/java/com/skyflow/detect/DeidentifyFileSource.java new file mode 100644 index 00000000..f60439ac --- /dev/null +++ b/common/src/main/java/com/skyflow/detect/DeidentifyFileSource.java @@ -0,0 +1,15 @@ +package com.skyflow.detect; + +/** + * How the file of {@code deidentifyFile} is supplied, and how a run reports its outputs. Each SDK ships its own + * {@code com.skyflow.detect.DeidentifySourceType} enum implementing this interface with the sources its vault + * type accepts: skyvault {@code BASE64}, {@code SKYFLOW_ID}, {@code PRESIGNED_URL}; flowvault {@code BASE64}, + * {@code PRESIGNED_URL}. {@code getOutputType()} on {@code DeidentifyFileResponse} and {@code GetRunResponse} + * returns a constant of that same enum. The constant name is the wire value. + */ +public interface DeidentifyFileSource { + /** + * @return the wire value, which is the constant name + */ + String name(); +} diff --git a/common/src/main/java/com/skyflow/detect/EntityType.java b/common/src/main/java/com/skyflow/detect/EntityType.java index 99cbbd8a..82cae959 100644 --- a/common/src/main/java/com/skyflow/detect/EntityType.java +++ b/common/src/main/java/com/skyflow/detect/EntityType.java @@ -1,7 +1,7 @@ package com.skyflow.detect; /** - * Entity types Skyflow Detect can identify. Also used to target entities in {@link RedactionLevel}. + * Entity types Skyflow Detect can identify. Also used to target entities in {@code RedactionLevel}. */ public enum EntityType { NONE, diff --git a/common/src/main/java/com/skyflow/detect/GetRunResponse.java b/common/src/main/java/com/skyflow/detect/GetRunResponse.java index e8ffbc27..c892355e 100644 --- a/common/src/main/java/com/skyflow/detect/GetRunResponse.java +++ b/common/src/main/java/com/skyflow/detect/GetRunResponse.java @@ -8,12 +8,12 @@ public final class GetRunResponse { private final String runId; private final DetectRunStatus status; - private final DataSourceType outputType; + private final DeidentifyFileSource outputType; private final List output; private final Metrics metrics; private final String message; - public GetRunResponse(String runId, DetectRunStatus status, DataSourceType outputType, List output, Metrics metrics, String message) { + public GetRunResponse(String runId, DetectRunStatus status, DeidentifyFileSource outputType, List output, Metrics metrics, String message) { this.runId = runId; this.status = status; this.outputType = outputType; @@ -37,9 +37,9 @@ public DetectRunStatus getStatus() { } /** - * @return How output files are delivered. Null while the run is pending. + * @return How output files are delivered. Null while the run is pending. A constant of this SDK's {@code DeidentifySourceType}: {@code BASE64} or {@code PRESIGNED_URL}. */ - public DataSourceType getOutputType() { + public DeidentifyFileSource getOutputType() { return outputType; } diff --git a/common/src/main/java/com/skyflow/detect/RedactionLevel.java b/common/src/main/java/com/skyflow/detect/RedactionLevel.java deleted file mode 100644 index b10d1806..00000000 --- a/common/src/main/java/com/skyflow/detect/RedactionLevel.java +++ /dev/null @@ -1,115 +0,0 @@ -package com.skyflow.detect; - -import com.skyflow.enums.RedactionType; - -/** - * Controls how one group of tokens is rendered when re-identifying. Supply exactly one source (entityName or tokenGroupName) and exactly one replacement (redactionType or redactionPattern). - */ -public final class RedactionLevel { - private final EntityType entityName; - private final RedactionType redactionType; - private final String tokenGroupName; - private final String redactionPattern; - - private RedactionLevel(RedactionLevelBuilder builder) { - this.entityName = builder.entityName; - this.redactionType = builder.redactionType; - this.tokenGroupName = builder.tokenGroupName; - this.redactionPattern = builder.redactionPattern; - } - - /** - * @return a new builder - */ - public static RedactionLevelBuilder builder() { - return new RedactionLevelBuilder(); - } - - /** - * @return Entity to target. skyvault vaults. - */ - public EntityType getEntityName() { - return entityName; - } - - /** - * @return Replacement rendering. skyvault vaults. - */ - public RedactionType getRedactionType() { - return redactionType; - } - - /** - * @return Token group to target. flowvault vaults. - */ - public String getTokenGroupName() { - return tokenGroupName; - } - - /** - * @return Named redaction pattern to apply. flowvault vaults. - */ - public String getRedactionPattern() { - return redactionPattern; - } - - @Override - public String toString() { - return "RedactionLevel{" + - "entityName=" + entityName + - ", redactionType=" + redactionType + - ", tokenGroupName=" + tokenGroupName + - ", redactionPattern=" + redactionPattern + - '}'; - } - - public static final class RedactionLevelBuilder { - private EntityType entityName; - private RedactionType redactionType; - private String tokenGroupName; - private String redactionPattern; - - private RedactionLevelBuilder() { - } - - /** - * @param entityName Entity to target. skyvault vaults. - * @return this builder - */ - public RedactionLevelBuilder entityName(EntityType entityName) { - this.entityName = entityName; - return this; - } - - /** - * @param redactionType Replacement rendering. skyvault vaults. - * @return this builder - */ - public RedactionLevelBuilder redactionType(RedactionType redactionType) { - this.redactionType = redactionType; - return this; - } - - /** - * @param tokenGroupName Token group to target. flowvault vaults. - * @return this builder - */ - public RedactionLevelBuilder tokenGroupName(String tokenGroupName) { - this.tokenGroupName = tokenGroupName; - return this; - } - - /** - * @param redactionPattern Named redaction pattern to apply. flowvault vaults. - * @return this builder - */ - public RedactionLevelBuilder redactionPattern(String redactionPattern) { - this.redactionPattern = redactionPattern; - return this; - } - - public RedactionLevel build() { - return new RedactionLevel(this); - } - } -} diff --git a/common/src/main/java/com/skyflow/detect/RedactionRule.java b/common/src/main/java/com/skyflow/detect/RedactionRule.java new file mode 100644 index 00000000..19d20716 --- /dev/null +++ b/common/src/main/java/com/skyflow/detect/RedactionRule.java @@ -0,0 +1,26 @@ +package com.skyflow.detect; + +/** + * One re-identification rendering rule, the element of {@code redactionLevel} on {@code ReidentifyStringRequest} + * and {@code ReidentifyFileRequest}. Each SDK ships its own {@code com.skyflow.detect.RedactionLevel} with the + * pair of fields its vault type uses: skyvault names an {@code entityName} and a {@code redactionType}; flowvault + * names a {@code tokenGroupName} and a {@code redactionPattern}. Only those classes can extend this type. + */ +public abstract class RedactionRule { + + RedactionRule() { + // subclasses live in this package, one per SDK + } + + /** @return the wire {@code entityName}, or null when this SDK's rule does not carry one */ + abstract String apiEntityName(); + + /** @return the wire {@code redactionType}, or null when this SDK's rule does not carry one */ + abstract String apiRedactionType(); + + /** @return the wire {@code tokenGroupName}, or null when this SDK's rule does not carry one */ + abstract String apiTokenGroupName(); + + /** @return the wire {@code redactionPattern}, or null when this SDK's rule does not carry one */ + abstract String apiRedactionPattern(); +} diff --git a/common/src/main/java/com/skyflow/detect/RedactionRules.java b/common/src/main/java/com/skyflow/detect/RedactionRules.java new file mode 100644 index 00000000..9a9817d6 --- /dev/null +++ b/common/src/main/java/com/skyflow/detect/RedactionRules.java @@ -0,0 +1,36 @@ +package com.skyflow.detect; + +import com.skyflow.generated.common.rest.types.RedactionLevel; +import com.skyflow.generated.common.rest.types.RedactionLevelEntityName; +import com.skyflow.generated.common.rest.types.RedactionLevelRedactionType; + +/** + * Internal: turns an SDK {@link RedactionRule} into the API's {@code RedactionLevel}, which carries all four + * fields for every vault type. Not part of the public contract. + */ +public final class RedactionRules { + + private RedactionRules() { + } + + /** + * @param rule a rule built with this SDK's {@code RedactionLevel} + * @return the wire representation with only the fields the rule carries + */ + public static RedactionLevel toApi(RedactionRule rule) { + RedactionLevel.Builder builder = RedactionLevel.builder(); + if (rule.apiEntityName() != null) { + builder.entityName(RedactionLevelEntityName.valueOf(rule.apiEntityName())); + } + if (rule.apiRedactionType() != null) { + builder.redactionType(RedactionLevelRedactionType.valueOf(rule.apiRedactionType())); + } + if (rule.apiTokenGroupName() != null) { + builder.tokenGroupName(rule.apiTokenGroupName()); + } + if (rule.apiRedactionPattern() != null) { + builder.redactionPattern(rule.apiRedactionPattern()); + } + return builder.build(); + } +} diff --git a/common/src/main/java/com/skyflow/detect/ReidentifyFileRequest.java b/common/src/main/java/com/skyflow/detect/ReidentifyFileRequest.java index 9aa7135e..82b1aff4 100644 --- a/common/src/main/java/com/skyflow/detect/ReidentifyFileRequest.java +++ b/common/src/main/java/com/skyflow/detect/ReidentifyFileRequest.java @@ -3,15 +3,15 @@ import java.util.List; /** - * Request for {@code reidentifyFile}. Supply the file through exactly one {@link DataSourceType}: - * the base64-encoded content, the skyflow id of a vault record that holds the file, or an S3 - * presigned URL. The vault id is filled in by the controller. + * Request for {@code reidentifyFile}. Supply the file through this SDK's {@code ReidentifySourceType} + * ({@code BASE64} or {@code PRESIGNED_URL}; re-identification does not accept a skyflow id) and the matching + * {@code value}. */ public final class ReidentifyFileRequest { - private final DataSourceType dataSource; + private final ReidentifyFileSource dataSource; private final String value; private final FileDataFormat dataFormat; - private final List redactionLevel; + private final List redactionLevel; private ReidentifyFileRequest(ReidentifyFileRequestBuilder builder) { this.dataSource = builder.dataSource; @@ -28,14 +28,15 @@ public static ReidentifyFileRequestBuilder builder() { } /** - * @return How {@code value} supplies the file. Required. + * @return How {@code value} supplies the file: this SDK's {@code ReidentifySourceType} (skyvault and flowvault: + * BASE64, PRESIGNED_URL; a skyflow id is not offered for re-identification). Required. */ - public DataSourceType getDataSource() { + public ReidentifyFileSource getDataSource() { return dataSource; } /** - * @return Base64 content, skyflow id or presigned URL of the file, according to {@code dataSource}. Required. + * @return Base64 content or presigned URL of the file, according to {@code dataSource}. Required. */ public String getValue() { return value; @@ -50,9 +51,9 @@ public FileDataFormat getDataFormat() { } /** - * @return Optional per-group rendering rules. + * @return Optional per-group rendering rules. Built with this SDK's {@code RedactionLevel}. */ - public List getRedactionLevel() { + public List getRedactionLevel() { return redactionLevel; } @@ -67,25 +68,26 @@ public String toString() { } public static final class ReidentifyFileRequestBuilder { - private DataSourceType dataSource; + private ReidentifyFileSource dataSource; private String value; private FileDataFormat dataFormat; - private List redactionLevel; + private List redactionLevel; private ReidentifyFileRequestBuilder() { } /** - * @param dataSource How {@code value} supplies the file. Required. + * @param dataSource How {@code value} supplies the file: this SDK's {@code ReidentifySourceType} (skyvault and + * flowvault: BASE64, PRESIGNED_URL; a skyflow id is not offered for re-identification). Required. * @return this builder */ - public ReidentifyFileRequestBuilder dataSource(DataSourceType dataSource) { + public ReidentifyFileRequestBuilder dataSource(ReidentifyFileSource dataSource) { this.dataSource = dataSource; return this; } /** - * @param value Base64 content, skyflow id or presigned URL of the file, according to {@code dataSource}. Required. + * @param value Base64 content or presigned URL of the file, according to {@code dataSource}. Required. * @return this builder */ public ReidentifyFileRequestBuilder value(String value) { @@ -103,10 +105,10 @@ public ReidentifyFileRequestBuilder dataFormat(FileDataFormat dataFormat) { } /** - * @param redactionLevel Optional per-group rendering rules. + * @param redactionLevel Optional rendering rules, built with this SDK's {@code RedactionLevel}. * @return this builder */ - public ReidentifyFileRequestBuilder redactionLevel(List redactionLevel) { + public ReidentifyFileRequestBuilder redactionLevel(List redactionLevel) { this.redactionLevel = redactionLevel; return this; } diff --git a/common/src/main/java/com/skyflow/detect/ReidentifyFileResponse.java b/common/src/main/java/com/skyflow/detect/ReidentifyFileResponse.java index 17d32f3e..50938fbb 100644 --- a/common/src/main/java/com/skyflow/detect/ReidentifyFileResponse.java +++ b/common/src/main/java/com/skyflow/detect/ReidentifyFileResponse.java @@ -8,11 +8,11 @@ */ public final class ReidentifyFileResponse { private final DetectRunStatus status; - private final DataSourceType outputType; + private final ReidentifyFileSource outputType; private final List output; private final Metrics metrics; - public ReidentifyFileResponse(DetectRunStatus status, DataSourceType outputType, List output, Metrics metrics) { + public ReidentifyFileResponse(DetectRunStatus status, ReidentifyFileSource outputType, List output, Metrics metrics) { this.status = status; this.outputType = outputType; this.output = output; @@ -27,9 +27,9 @@ public DetectRunStatus getStatus() { } /** - * @return How output files are delivered, or null when not reported. + * @return How output files are delivered, or null when not reported. A constant of this SDK's {@code ReidentifySourceType}: {@code BASE64} or {@code PRESIGNED_URL}. */ - public DataSourceType getOutputType() { + public ReidentifyFileSource getOutputType() { return outputType; } diff --git a/common/src/main/java/com/skyflow/detect/ReidentifyFileSource.java b/common/src/main/java/com/skyflow/detect/ReidentifyFileSource.java new file mode 100644 index 00000000..d0f54ab3 --- /dev/null +++ b/common/src/main/java/com/skyflow/detect/ReidentifyFileSource.java @@ -0,0 +1,14 @@ +package com.skyflow.detect; + +/** + * How the file of {@code reidentifyFile} is supplied, and how its response reports the output. Each SDK ships + * its own {@code com.skyflow.detect.ReidentifySourceType} enum implementing this interface, with {@code BASE64} + * and {@code PRESIGNED_URL} in both SDKs. {@code getOutputType()} on {@code ReidentifyFileResponse} returns a + * constant of that same enum. The constant name is the wire value. + */ +public interface ReidentifyFileSource { + /** + * @return the wire value, which is the constant name + */ + String name(); +} diff --git a/common/src/main/java/com/skyflow/detect/ReidentifyStringRequest.java b/common/src/main/java/com/skyflow/detect/ReidentifyStringRequest.java index 23626297..acabf2bb 100644 --- a/common/src/main/java/com/skyflow/detect/ReidentifyStringRequest.java +++ b/common/src/main/java/com/skyflow/detect/ReidentifyStringRequest.java @@ -7,7 +7,7 @@ */ public final class ReidentifyStringRequest { private final String text; - private final List redactionLevel; + private final List redactionLevel; private ReidentifyStringRequest(ReidentifyStringRequestBuilder builder) { this.text = builder.text; @@ -29,9 +29,9 @@ public String getText() { } /** - * @return Optional per-group rendering rules. + * @return Optional per-group rendering rules. Built with this SDK's {@code RedactionLevel}. */ - public List getRedactionLevel() { + public List getRedactionLevel() { return redactionLevel; } @@ -45,7 +45,7 @@ public String toString() { public static final class ReidentifyStringRequestBuilder { private String text; - private List redactionLevel; + private List redactionLevel; private ReidentifyStringRequestBuilder() { } @@ -60,10 +60,10 @@ public ReidentifyStringRequestBuilder text(String text) { } /** - * @param redactionLevel Optional per-group rendering rules. + * @param redactionLevel Optional rendering rules, built with this SDK's {@code RedactionLevel}. * @return this builder */ - public ReidentifyStringRequestBuilder redactionLevel(List redactionLevel) { + public ReidentifyStringRequestBuilder redactionLevel(List redactionLevel) { this.redactionLevel = redactionLevel; return this; } diff --git a/common/src/main/java/com/skyflow/detect/internal/DeidentifyFileFlow.java b/common/src/main/java/com/skyflow/detect/internal/DeidentifyFileFlow.java index 4d825983..da64c9b2 100644 --- a/common/src/main/java/com/skyflow/detect/internal/DeidentifyFileFlow.java +++ b/common/src/main/java/com/skyflow/detect/internal/DeidentifyFileFlow.java @@ -1,5 +1,6 @@ package com.skyflow.detect.internal; +import java.io.File; import com.skyflow.detect.DeidentifyFileRequest; import com.skyflow.detect.DeidentifyFileResponse; import com.skyflow.detect.DetectRunStatus; @@ -32,8 +33,9 @@ static DeidentifyFileResponse complete(DeidentifyFileRequest request, String run } GetRunResponse run = RunPoller.poll(runId, request.getPollOptions(), fetcher, sleeper); DeidentifyFileResponse response = DetectResponseMapper.toDeidentifyFileResponse(runId, run); - if (response.getStatus() == DetectRunStatus.SUCCESS && request.getFile() != null) { - FileOutputWriter.write(response.getOutput(), response.getOutputType(), request.getFile().getName(), + File localFile = DetectRequestMapper.localFile(request); + if (response.getStatus() == DetectRunStatus.SUCCESS && localFile != null) { + FileOutputWriter.write(response.getOutput(), response.getOutputType(), localFile.getName(), request.getOutputDirectory()); } return response; diff --git a/common/src/main/java/com/skyflow/detect/internal/DetectRequestMapper.java b/common/src/main/java/com/skyflow/detect/internal/DetectRequestMapper.java index c11a9bab..9e2b42cf 100644 --- a/common/src/main/java/com/skyflow/detect/internal/DetectRequestMapper.java +++ b/common/src/main/java/com/skyflow/detect/internal/DetectRequestMapper.java @@ -11,7 +11,6 @@ import com.skyflow.detect.Bleep; import com.skyflow.detect.CheckGuardrailsRequest; import com.skyflow.detect.CustomType; -import com.skyflow.detect.DataSourceType; import com.skyflow.detect.DeidentifyFileRequest; import com.skyflow.detect.DeidentifyStringRequest; import com.skyflow.detect.FileDataFormat; @@ -24,7 +23,8 @@ import com.skyflow.detect.Media; import com.skyflow.detect.ObjectEntity; import com.skyflow.detect.Pdf; -import com.skyflow.detect.RedactionLevel; +import com.skyflow.detect.RedactionRule; +import com.skyflow.detect.RedactionRules; import com.skyflow.detect.ReidentifyStringRequest; import com.skyflow.detect.ShiftDates; import com.skyflow.detect.Transformation; @@ -58,8 +58,6 @@ import com.skyflow.generated.common.rest.types.EntityTransformation; import com.skyflow.generated.common.rest.types.ImageV2; import com.skyflow.generated.common.rest.types.ImageV2MaskingMethod; -import com.skyflow.generated.common.rest.types.RedactionLevelEntityName; -import com.skyflow.generated.common.rest.types.RedactionLevelRedactionType; import com.skyflow.generated.common.rest.types.TransformationShiftDates; /** @@ -94,11 +92,23 @@ static boolean sendsConfiguration(String configurationId, DetectConfiguration co * Builds the submit request. For file-based requests this reads and base64-encodes the file and infers * the format from its extension when none was given. */ + /** + * The local file a deidentifyFile request points at, from {@code file} or {@code filePath}, or null for a + * {@code value}-based request. + */ + public static File localFile(DeidentifyFileRequest request) { + if (request.getFile() != null) { + return request.getFile(); + } + String path = request.getFilePath(); + return path == null || path.trim().isEmpty() ? null : new File(path); + } + public static DeidentifyFileRequestV2 toDeidentifyFileRequest(DeidentifyFileRequest request, String vaultId) throws SkyflowException { String value = request.getValue(); FileDataFormat format = request.getDataFormat(); - File file = request.getFile(); + File file = localFile(request); if (file != null) { value = encodeFile(file); if (format == null) { @@ -106,9 +116,11 @@ public static DeidentifyFileRequestV2 toDeidentifyFileRequest(DeidentifyFileRequ } } // A local file is always sent as BASE64, whatever dataSource the builder was left with. - DataSourceType dataSource = file != null ? DataSourceType.BASE64 : request.getDataSource(); + DeidentifyFileRequestV2DataSource dataSource = file != null + ? DeidentifyFileRequestV2DataSource.BASE64 + : DeidentifyFileRequestV2DataSource.valueOf(request.getDataSource().name()); DeidentifyFileRequestV2._FinalStage builder = DeidentifyFileRequestV2.builder() - .dataSource(DeidentifyFileRequestV2DataSource.valueOf(dataSource.name())) + .dataSource(dataSource) .value(value); if (format != null) { // The wire value for a file format is the lower-case extension. @@ -397,27 +409,13 @@ static ImageV2 toImage(Image image) { return builder.build(); } - public static List toRedactionLevels(List levels) { + public static List toRedactionLevels(List levels) { List result = new ArrayList<>(); - for (RedactionLevel level : levels) { + for (RedactionRule level : levels) { if (level == null) { continue; } - com.skyflow.generated.common.rest.types.RedactionLevel.Builder builder = - com.skyflow.generated.common.rest.types.RedactionLevel.builder(); - if (level.getEntityName() != null) { - builder.entityName(RedactionLevelEntityName.valueOf(level.getEntityName().name())); - } - if (level.getRedactionType() != null) { - builder.redactionType(RedactionLevelRedactionType.valueOf(level.getRedactionType().name())); - } - if (level.getTokenGroupName() != null) { - builder.tokenGroupName(level.getTokenGroupName()); - } - if (level.getRedactionPattern() != null) { - builder.redactionPattern(level.getRedactionPattern()); - } - result.add(builder.build()); + result.add(RedactionRules.toApi(level)); } return result; } diff --git a/common/src/main/java/com/skyflow/detect/internal/DetectResponseMapper.java b/common/src/main/java/com/skyflow/detect/internal/DetectResponseMapper.java index 9994ab70..44a9dba6 100644 --- a/common/src/main/java/com/skyflow/detect/internal/DetectResponseMapper.java +++ b/common/src/main/java/com/skyflow/detect/internal/DetectResponseMapper.java @@ -1,7 +1,7 @@ package com.skyflow.detect.internal; import com.skyflow.detect.CheckGuardrailsResponse; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.DeidentifyFileSource; import com.skyflow.detect.DeidentifyFileResponse; import com.skyflow.detect.DeidentifyStringResponse; import com.skyflow.detect.DetectRunStatus; @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.function.Function; import java.util.Map; /** @@ -81,7 +82,12 @@ public static CheckGuardrailsResponse toCheckGuardrailsResponse(DetectGuardrails response.getDeniedTopic().orElse(null)); } - public static GetRunResponse toGetRunResponse(String runId, DetectRunsResponseV2 response) { + /** + * @param sourceResolver maps the wire outputType onto this SDK's {@code DeidentifySourceType}; returns null for an + * unknown value + */ + public static GetRunResponse toGetRunResponse(String runId, DetectRunsResponseV2 response, + Function sourceResolver) { List outputs = new ArrayList<>(); List apiOutputs = response.getOutput().orElse(null); if (apiOutputs != null) { @@ -95,7 +101,7 @@ public static GetRunResponse toGetRunResponse(String runId, DetectRunsResponseV2 runId, response.getStatus().map(s -> enumOrDefault(DetectRunStatus.class, s.toString(), DetectRunStatus.UNKNOWN)) .orElse(DetectRunStatus.UNKNOWN), - response.getOutputType().map(t -> enumOrDefault(DataSourceType.class, t.toString(), null)).orElse(null), + response.getOutputType().map(t -> sourceResolver.apply(t.toString())).orElse(null), Collections.unmodifiableList(outputs), response.getMetrics().map(DetectResponseMapper::toMetrics).orElse(null), response.getMessage().orElse(null)); diff --git a/common/src/main/java/com/skyflow/detect/internal/DetectValidations.java b/common/src/main/java/com/skyflow/detect/internal/DetectValidations.java index e7b19364..db3345fb 100644 --- a/common/src/main/java/com/skyflow/detect/internal/DetectValidations.java +++ b/common/src/main/java/com/skyflow/detect/internal/DetectValidations.java @@ -13,6 +13,8 @@ import com.skyflow.utils.BaseUtils; import com.skyflow.utils.logger.LogUtil; +import java.io.File; + /** * Client-side validation for Detect V2 requests. Internal to the SDK. @@ -38,20 +40,27 @@ public static void validateDeidentifyFileRequest(DeidentifyFileRequest request) if (request == null) { throw invalid(ErrorLogs.DEIDENTIFY_FILE_REQUEST_NULL, BaseErrorMessage.DeidentifyFileRequestNull); } - if (request.getFile() != null) { - if (!isBlank(request.getValue())) { - throw invalid(ErrorLogs.MULTIPLE_FILE_SOURCES_IN_DEIDENTIFY_FILE, BaseErrorMessage.MultipleFileSourcesInDeidentifyFile); - } - if (!request.getFile().isFile()) { - throw invalid(ErrorLogs.FILE_NOT_FOUND_IN_DEIDENTIFY_FILE, BaseErrorMessage.FileNotFoundToDeidentify); - } - } else { + // The file comes from exactly one of filePath, file or value; dataSource belongs to value only. + int sources = (request.getFilePath() != null ? 1 : 0) + (request.getFile() != null ? 1 : 0) + + (request.getValue() != null ? 1 : 0); + if (sources == 0) { + throw invalid(ErrorLogs.MISSING_FILE_SOURCE_IN_DEIDENTIFY_FILE, BaseErrorMessage.MissingFileSourceInDeidentifyFile); + } + if (sources > 1) { + throw invalid(ErrorLogs.MULTIPLE_FILE_SOURCES_IN_DEIDENTIFY_FILE, BaseErrorMessage.MultipleFileSourcesInDeidentifyFile); + } + if (request.getValue() != null) { if (request.getDataSource() == null) { throw invalid(ErrorLogs.INVALID_DATA_SOURCE_IN_DEIDENTIFY_FILE, BaseErrorMessage.InvalidDataSourceInDeidentifyFile); } if (isBlank(request.getValue())) { throw invalid(ErrorLogs.INVALID_VALUE_IN_DEIDENTIFY_FILE, BaseErrorMessage.InvalidValueInDeidentifyFile); } + } else { + File file = DetectRequestMapper.localFile(request); + if (file == null || !file.isFile()) { + throw invalid(ErrorLogs.FILE_NOT_FOUND_IN_DEIDENTIFY_FILE, BaseErrorMessage.FileNotFoundToDeidentify); + } } // configurationId and configuration are both optional on the API and are passed through as given; // with neither set the mapper sends a configuration carrying only the vault id. diff --git a/common/src/main/java/com/skyflow/detect/internal/FileOutputWriter.java b/common/src/main/java/com/skyflow/detect/internal/FileOutputWriter.java index f82f0957..73261728 100644 --- a/common/src/main/java/com/skyflow/detect/internal/FileOutputWriter.java +++ b/common/src/main/java/com/skyflow/detect/internal/FileOutputWriter.java @@ -1,6 +1,6 @@ package com.skyflow.detect.internal; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.DeidentifyFileSource; import com.skyflow.detect.FileOutput; import com.skyflow.errors.BaseErrorMessage; import com.skyflow.errors.SkyflowException; @@ -40,13 +40,13 @@ private FileOutputWriter() { * @return the files written, in output order; empty when the output type is not BASE64 * @throws SkyflowException if the directory cannot be created or a file cannot be decoded or written */ - public static List write(List outputs, DataSourceType outputType, String inputFileName, + public static List write(List outputs, DeidentifyFileSource outputType, String inputFileName, String outputDirectory) throws SkyflowException { List written = new ArrayList<>(); if (outputs == null || outputs.isEmpty()) { return written; } - if (outputType != DataSourceType.BASE64) { + if (outputType == null || !"BASE64".equals(outputType.name())) { LogUtil.printInfoLog(BaseUtils.parameterizedString(InfoLogs.OUTPUT_NOT_WRITTEN_FOR_OUTPUT_TYPE.getLog(), String.valueOf(outputType))); return written; diff --git a/common/src/main/java/com/skyflow/detect/internal/ReidentifyFileMapper.java b/common/src/main/java/com/skyflow/detect/internal/ReidentifyFileMapper.java index 188f0013..73448a3a 100644 --- a/common/src/main/java/com/skyflow/detect/internal/ReidentifyFileMapper.java +++ b/common/src/main/java/com/skyflow/detect/internal/ReidentifyFileMapper.java @@ -1,6 +1,6 @@ package com.skyflow.detect.internal; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.ReidentifyFileSource; import com.skyflow.detect.DetectRunStatus; import com.skyflow.detect.FileOutput; import com.skyflow.detect.ReidentifyFileRequest; @@ -13,6 +13,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.function.Function; /** * Converts the flowvault {@code reidentifyFile} models to and from the generated V2 API types. @@ -37,7 +38,12 @@ public static ReidentifyFileRequestV2 toApiRequest(ReidentifyFileRequest request return builder.build(); } - public static ReidentifyFileResponse toResponse(ReidentifyFileResponseV2 response) { + /** + * @param sourceResolver maps the wire outputType onto this SDK's {@code ReidentifySourceType}; returns null for an + * unknown value + */ + public static ReidentifyFileResponse toResponse(ReidentifyFileResponseV2 response, + Function sourceResolver) { List outputs = new ArrayList<>(); List apiOutputs = response.getOutput().orElse(null); if (apiOutputs != null) { @@ -50,7 +56,7 @@ public static ReidentifyFileResponse toResponse(ReidentifyFileResponseV2 respons return new ReidentifyFileResponse( response.getStatus().map(s -> enumOrDefault(DetectRunStatus.class, s.toString(), DetectRunStatus.UNKNOWN)) .orElse(DetectRunStatus.UNKNOWN), - response.getOutputType().map(t -> enumOrDefault(DataSourceType.class, t.toString(), null)).orElse(null), + response.getOutputType().map(t -> sourceResolver.apply(t.toString())).orElse(null), Collections.unmodifiableList(outputs), response.getMetrics().map(DetectResponseMapper::toMetrics).orElse(null)); } diff --git a/common/src/main/java/com/skyflow/errors/BaseErrorMessage.java b/common/src/main/java/com/skyflow/errors/BaseErrorMessage.java index ff51fb89..b3eb34c7 100644 --- a/common/src/main/java/com/skyflow/errors/BaseErrorMessage.java +++ b/common/src/main/java/com/skyflow/errors/BaseErrorMessage.java @@ -184,17 +184,18 @@ public enum BaseErrorMessage { ReidentifyStringRequestNull("%s0 Validation error. ReidentifyString request cannot be null."), InvalidTextInReidentifyString("%s0 Validation error. Invalid text in reidentify string request. Specify a non-empty text."), DeidentifyFileRequestNull("%s0 Validation error. DeidentifyFile request cannot be null."), - InvalidDataSourceInDeidentifyFile("%s0 Validation error. Invalid dataSource in deidentify file request. Specify BASE64, SKYFLOW_ID or PRESIGNED_URL, or supply a file."), - InvalidValueInDeidentifyFile("%s0 Validation error. Invalid value in deidentify file request. Specify the non-empty base64 content, skyflow id or presigned URL of the file, or supply a file."), - MultipleFileSourcesInDeidentifyFile("%s0 Validation error. Specify either value or file in deidentify file request, not both."), + InvalidDataSourceInDeidentifyFile("%s0 Validation error. Invalid dataSource in deidentify file request. Specify a DeidentifySourceType."), + InvalidValueInDeidentifyFile("%s0 Validation error. Invalid value in deidentify file request. Specify the non-empty content or reference that dataSource describes."), + MultipleFileSourcesInDeidentifyFile("%s0 Validation error. Specify only one of filePath, file or value in deidentify file request."), + MissingFileSourceInDeidentifyFile("%s0 Validation error. Specify one of filePath, file or value in deidentify file request."), InvalidWaitTimeInPollOptions("%s0 Validation error. waitTime in pollOptions must be between 1 and 300 seconds."), InvalidMaxAttemptsInPollOptions("%s0 Validation error. maxAttempts in pollOptions must be between 1 and 23."), OutputDirectoryWithoutPollOptions("%s0 Validation error. outputDirectory in deidentify file request requires pollOptions."), FailedToWriteOutputFile("%s0 Failed to write the processed output to %s1."), GetRunRequestNull("%s0 Validation error. GetRun request cannot be null."), ReidentifyFileRequestNull("%s0 Validation error. ReidentifyFile request cannot be null."), - InvalidDataSourceInReidentifyFile("%s0 Validation error. Invalid dataSource in reidentify file request. Specify BASE64, SKYFLOW_ID or PRESIGNED_URL."), - InvalidValueInReidentifyFile("%s0 Validation error. Invalid value in reidentify file request. Specify the non-empty base64 content, skyflow id or presigned URL of the file."), + InvalidDataSourceInReidentifyFile("%s0 Validation error. Invalid dataSource in reidentify file request. Specify a ReidentifySourceType."), + InvalidValueInReidentifyFile("%s0 Validation error. Invalid value in reidentify file request. Specify the non-empty base64 content or presigned URL of the file."), InvalidRunIdInGetRun("%s0 Validation error. Invalid runId in get run request. Specify a non-empty run id."), CheckGuardrailsRequestNull("%s0 Validation error. CheckGuardrails request cannot be null."), InvalidTextInCheckGuardrails("%s0 Validation error. Invalid text in check guardrails request. Specify a non-empty text."), diff --git a/common/src/main/java/com/skyflow/logs/ErrorLogs.java b/common/src/main/java/com/skyflow/logs/ErrorLogs.java index 154c26e8..5d5ce098 100644 --- a/common/src/main/java/com/skyflow/logs/ErrorLogs.java +++ b/common/src/main/java/com/skyflow/logs/ErrorLogs.java @@ -14,9 +14,10 @@ public enum ErrorLogs { INVALID_TEXT_IN_REIDENTIFY_STRING("Invalid %s1 request. The text field is required and must be a non-empty string. Specify a valid text."), REIDENTIFY_STRING_REQUEST_REJECTED("ReIdentify string request resulted in failure."), DEIDENTIFY_FILE_REQUEST_NULL("Invalid %s1 request. DeidentifyFile request can not be null."), - INVALID_DATA_SOURCE_IN_DEIDENTIFY_FILE("Invalid %s1 request. The dataSource field is required unless a file is supplied."), - INVALID_VALUE_IN_DEIDENTIFY_FILE("Invalid %s1 request. The value field is required and must be a non-empty string unless a file is supplied."), - MULTIPLE_FILE_SOURCES_IN_DEIDENTIFY_FILE("Invalid %s1 request. Specify either value or file, not both."), + INVALID_DATA_SOURCE_IN_DEIDENTIFY_FILE("Invalid %s1 request. The dataSource field is required with value. Specify a DeidentifySourceType."), + INVALID_VALUE_IN_DEIDENTIFY_FILE("Invalid %s1 request. The value field must be a non-empty string."), + MULTIPLE_FILE_SOURCES_IN_DEIDENTIFY_FILE("Invalid %s1 request. Specify only one of filePath, file or value."), + MISSING_FILE_SOURCE_IN_DEIDENTIFY_FILE("Invalid %s1 request. Specify one of filePath, file or value."), FILE_NOT_FOUND_IN_DEIDENTIFY_FILE("Invalid %s1 request. The file to deidentify was not found."), FAILED_TO_ENCODE_FILE_IN_DEIDENTIFY_FILE("Invalid %s1 request. Failed to read and encode the file to deidentify."), INVALID_WAIT_TIME_IN_POLL_OPTIONS("Invalid %s1 request. waitTime in pollOptions must be between 1 and 300."), @@ -26,8 +27,8 @@ public enum ErrorLogs { OUTPUT_DIRECTORY_NOT_CREATED("Could not create output directory %s1."), OUTPUT_FILE_NOT_WRITTEN("Could not write processed output to %s1."), REIDENTIFY_FILE_REQUEST_NULL("Invalid %s1 request. ReidentifyFile request can not be null."), - INVALID_DATA_SOURCE_IN_REIDENTIFY_FILE("Invalid %s1 request. The dataSource field is required. Specify BASE64, SKYFLOW_ID or PRESIGNED_URL."), - INVALID_VALUE_IN_REIDENTIFY_FILE("Invalid %s1 request. The value field is required and must be a non-empty string. Specify the base64 content, skyflow id or presigned URL of the file."), + INVALID_DATA_SOURCE_IN_REIDENTIFY_FILE("Invalid %s1 request. The dataSource field is required. Specify a ReidentifySourceType."), + INVALID_VALUE_IN_REIDENTIFY_FILE("Invalid %s1 request. The value field is required and must be a non-empty string. Specify the base64 content or presigned URL of the file."), REIDENTIFY_FILE_REQUEST_REJECTED("ReIdentify file request resulted in failure."), // Client initialization VAULT_CONFIG_EXISTS("Vault config with vault ID %s1 already exists."), diff --git a/common/src/main/java/com/skyflow/logs/InfoLogs.java b/common/src/main/java/com/skyflow/logs/InfoLogs.java index 3c332232..aa970441 100644 --- a/common/src/main/java/com/skyflow/logs/InfoLogs.java +++ b/common/src/main/java/com/skyflow/logs/InfoLogs.java @@ -51,6 +51,10 @@ public enum InfoLogs { GET_SIGNED_DATA_TOKENS_TRIGGERED("getSignedDataTokens method triggered."), GET_SIGNED_DATA_TOKEN_SUCCESS("Signed data tokens generated."), REUSE_BEARER_TOKEN("Reusing bearer token."), + BEARER_TOKEN_REJECTED_REGENERATING("Bearer token rejected with 401 and is past expiry; regenerating once and resending the request."), + BEARER_TOKEN_REJECTED_REUSING_REGENERATED("Bearer token rejected with 401; reusing the token another request already regenerated."), + BEARER_TOKEN_REJECTED_NOT_DUE("Bearer token rejected with 401 but is not near expiry; not retrying."), + BEARER_TOKEN_REJECTED_REGENERATION_ALREADY_FAILED("Bearer token rejected with 401; regeneration for this token already failed, not retrying."), USE_CLIENT_PROVIDED_BEARER_TOKEN("Using bearer token provided by client."), USE_API_KEY("Using api key."), REUSE_API_KEY("Reusing api key."), diff --git a/common/src/main/java/com/skyflow/logs/WarningLogs.java b/common/src/main/java/com/skyflow/logs/WarningLogs.java index 1eb1bcd8..9d001694 100644 --- a/common/src/main/java/com/skyflow/logs/WarningLogs.java +++ b/common/src/main/java/com/skyflow/logs/WarningLogs.java @@ -8,7 +8,9 @@ public enum WarningLogs { EMPTY_DELETE_TOKENS_RESPONSE("DeleteTokens response did not include any token results."), INCOMPLETE_DELETE_TOKENS_RESPONSE("DeleteTokens response did not account for all requested tokens."), EMPTY_TOKENIZE_RESPONSE("Tokenize response did not include any record results."), - INCOMPLETE_TOKENIZE_RESPONSE("Tokenize response did not account for all requested records.") + INCOMPLETE_TOKENIZE_RESPONSE("Tokenize response did not account for all requested records."), + BEARER_TOKEN_REGENERATION_AFTER_401_FAILED("Bearer token regeneration after 401 failed; returning the 401 response."), + BEARER_TOKEN_REGENERATION_RETURNED_REJECTED_TOKEN("Bearer token regeneration after 401 returned the rejected token; not resending.") ; private final String log; diff --git a/common/src/main/java/com/skyflow/serviceaccount/util/Token.java b/common/src/main/java/com/skyflow/serviceaccount/util/Token.java index 19edf0c8..724e902f 100644 --- a/common/src/main/java/com/skyflow/serviceaccount/util/Token.java +++ b/common/src/main/java/com/skyflow/serviceaccount/util/Token.java @@ -40,6 +40,29 @@ public static boolean isExpired(String token) { return currentTime > expiryTime; } + /** + * Returns true when the token's {@code exp} claim is at or before now plus {@code windowSeconds}, + * that is, the token has expired or will within the window. Unlike {@link #isExpired(String)}, + * a token that is empty, cannot be decoded, or carries no {@code exp} is reported as + * not expiring: callers use this to decide whether a {@code 401} was caused by expiry, + * and a malformed token is not an expiry. + */ + public static boolean isExpiringWithin(String token, long windowSeconds) { + if (token == null || token.trim().isEmpty()) { + return false; + } + try { + JsonElement exp = decoded(token).get("exp"); + if (exp == null) { + return false; + } + long currentTime = new Date().getTime() / 1000; + return exp.getAsLong() <= currentTime + windowSeconds; + } catch (SkyflowException | RuntimeException e) { + return false; + } + } + static JsonObject decoded(String encodedToken) throws JsonSyntaxException, SkyflowException { String[] split = encodedToken.split("\\."); if (split.length < 3) { diff --git a/common/src/main/java/com/skyflow/vault/controller/BaseDetectController.java b/common/src/main/java/com/skyflow/vault/controller/BaseDetectController.java index dc605441..aacb2c53 100644 --- a/common/src/main/java/com/skyflow/vault/controller/BaseDetectController.java +++ b/common/src/main/java/com/skyflow/vault/controller/BaseDetectController.java @@ -6,6 +6,8 @@ import com.skyflow.detect.CheckGuardrailsRequest; import com.skyflow.detect.CheckGuardrailsResponse; import com.skyflow.detect.DeidentifyFileRequest; +import com.skyflow.detect.ReidentifyFileSource; +import com.skyflow.detect.DeidentifyFileSource; import com.skyflow.detect.DeidentifyFileResponse; import com.skyflow.detect.DeidentifyStringRequest; import com.skyflow.detect.DeidentifyStringResponse; @@ -89,6 +91,35 @@ protected BaseDetectController(V vaultConfig, BaseCredentials credentials) { */ protected abstract String resolveVaultUrl() throws SkyflowException; + /** + * Maps a run's wire {@code outputType} onto this SDK's {@code DeidentifySourceType}, so that + * {@code getOutputType()} returns the same enum the caller used on the request. + * + * @param wireValue the API value, for example {@code BASE64} + * @return the matching constant, or null for a value this SDK does not know + */ + protected abstract DeidentifyFileSource toDeidentifySource(String wireValue); + + /** + * Maps a re-identify response's wire {@code outputType} onto this SDK's {@code ReidentifySourceType}. + * + * @param wireValue the API value, for example {@code BASE64} + * @return the matching constant, or null for a value this SDK does not know + */ + protected abstract ReidentifyFileSource toReidentifySource(String wireValue); + + /** {@link Enum#valueOf} that returns null instead of throwing for an unknown wire value. */ + protected static > E enumOrNull(Class type, String wireValue) { + if (wireValue == null) { + return null; + } + try { + return Enum.valueOf(type, wireValue); + } catch (IllegalArgumentException unknown) { + return null; + } + } + /** * @return JSON sent in the {@code sky-metadata} header. Override to report SDK name and version. */ @@ -271,7 +302,7 @@ public DeidentifyFileResponse deidentifyFile(DeidentifyFileRequest request) thro * Replaces tokens in a file with their original values or a redacted rendering. Synchronous: * the re-identified file is returned in the response rather than through a run id. * - * @param request the file (base64, skyflow id or presigned URL) plus optional per-group redaction rules + * @param request the file (base64 or presigned URL) plus optional per-group redaction rules * @return status, output files and metrics * @throws SkyflowException on validation failure, credential failure or an API error */ @@ -284,7 +315,7 @@ public ReidentifyFileResponse reidentifyFile(ReidentifyFileRequest request) thro try { ReidentifyFileResponseV2 apiResponse = filesClient().reidentifyFileV2(apiRequest, requestOptions(token)); LogUtil.printInfoLog(InfoLogs.REIDENTIFY_FILE_REQUEST_RESOLVED.getLog()); - ReidentifyFileResponse response = ReidentifyFileMapper.toResponse(apiResponse); + ReidentifyFileResponse response = ReidentifyFileMapper.toResponse(apiResponse, this::toReidentifySource); LogUtil.printInfoLog(InfoLogs.REIDENTIFY_FILE_SUCCESS.getLog()); return response; } catch (ApiClientApiException e) { @@ -312,7 +343,7 @@ public GetRunResponse getRun(GetRunRequest request) throws SkyflowException { try { DetectRunsResponseV2 apiResponse = filesClient().getRunV2(runId, apiRequest, requestOptions(token)); LogUtil.printInfoLog(InfoLogs.GET_RUN_REQUEST_RESOLVED.getLog()); - GetRunResponse response = DetectResponseMapper.toGetRunResponse(runId, apiResponse); + GetRunResponse response = DetectResponseMapper.toGetRunResponse(runId, apiResponse, this::toDeidentifySource); LogUtil.printInfoLog(InfoLogs.GET_RUN_SUCCESS.getLog()); return response; } catch (ApiClientApiException e) { diff --git a/common/src/test/java/com/skyflow/BaseVaultClientTests.java b/common/src/test/java/com/skyflow/BaseVaultClientTests.java index 278eb353..04cb71ee 100644 --- a/common/src/test/java/com/skyflow/BaseVaultClientTests.java +++ b/common/src/test/java/com/skyflow/BaseVaultClientTests.java @@ -4,8 +4,11 @@ import com.skyflow.config.BaseVaultConfig; import com.skyflow.errors.BaseErrorMessage; import com.skyflow.errors.SkyflowException; +import com.skyflow.enums.LogLevel; import com.skyflow.logs.ErrorLogs; +import com.skyflow.logs.WarningLogs; import com.skyflow.utils.BaseConstants; +import com.skyflow.utils.logger.LogUtil; import okhttp3.Call; import okhttp3.Connection; import okhttp3.Interceptor; @@ -26,8 +29,20 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; public class BaseVaultClientTests { @@ -301,4 +316,481 @@ public void testSetBearerToken_expiredToken_regeneratesToken() throws SkyflowExc Assert.assertEquals("x.eyJleHAiOjk5OTk5OTk5OTl9.y", client.token); Assert.assertNotEquals("", client.token); } + + // ── refresh-and-resend on 401 ──────────────────────────────────────────── + + private static String jwtExpiringIn(long seconds) { + Base64.Encoder enc = Base64.getUrlEncoder().withoutPadding(); + String header = enc.encodeToString("{\"alg\":\"none\"}".getBytes(StandardCharsets.UTF_8)); + String payload = enc.encodeToString(("{\"exp\":" + (System.currentTimeMillis() / 1000 + seconds) + "}") + .getBytes(StandardCharsets.UTF_8)); + return header + "." + payload + ".sig"; + } + + /** A client whose token generation is scripted, so no service-account file is needed. */ + private static final class ScriptedClient extends BaseVaultClient { + final AtomicInteger generations = new AtomicInteger(); + final java.util.List script = new ArrayList<>(); + SkyflowException failure; + + ScriptedClient(BaseCredentials credentials) { + super(new BaseVaultConfig(), credentials); + this.finalCredentials = credentials; + } + + @Override + protected String generateBearerToken(BaseCredentials credentials) throws SkyflowException { + int n = generations.getAndIncrement(); + if (failure != null) { + throw failure; + } + return script.get(Math.min(n, script.size() - 1)); + } + } + + private static BaseCredentials mintableCredentials() { + BaseCredentials credentials = new BaseCredentials(); + credentials.setCredentialsString("{\"clientID\":\"c\"}"); + return credentials; + } + + @Test + public void testRefreshAfter401_expiredSdkTokenIsRegeneratedOnceAndCached() { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = rejected; + client.script.add("fresh-token"); + + Assert.assertEquals("fresh-token", client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertEquals("fresh-token", client.token); + Assert.assertEquals(1, client.generations.get()); + } + + @Test + public void testRefreshAfter401_tokenInsideWindowCountsAsExpired() { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(30); + client.token = rejected; + client.script.add("fresh-token"); + + Assert.assertEquals("fresh-token", client.refreshBearerTokenAfterUnauthorized(rejected)); + } + + @Test + public void testRefreshAfter401_tokenWithLifeLeftIsNotRetried() { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(600); + client.token = rejected; + client.script.add("fresh-token"); + + Assert.assertNull("a 401 on a live token is not expiry", client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertEquals("must not spend a mint on a non-expiry 401", 0, client.generations.get()); + Assert.assertEquals(rejected, client.token); + } + + @Test + public void testRefreshAfter401_staticTokenIsNeverRegenerated() { + BaseCredentials credentials = new BaseCredentials(); + String rejected = jwtExpiringIn(-2); + credentials.setToken(rejected); + ScriptedClient client = new ScriptedClient(credentials); + client.token = rejected; + client.script.add("fresh-token"); + + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertEquals(0, client.generations.get()); + } + + @Test + public void testRefreshAfter401_apiKeyIsNeverRegenerated() { + BaseCredentials credentials = new BaseCredentials(); + credentials.setApiKey("sky-test-key"); + ScriptedClient client = new ScriptedClient(credentials); + client.token = "sky-test-key"; + client.script.add("fresh-token"); + + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized("sky-test-key")); + Assert.assertEquals(0, client.generations.get()); + } + + @Test + public void testRefreshAfter401_noCredentialsResolvedYet() { + ScriptedClient client = new ScriptedClient(null); + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(jwtExpiringIn(-2))); + } + + @Test + public void testRefreshAfter401_generationFailureReturnsNullAndKeepsToken() { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = rejected; + client.failure = new SkyflowException("auth service down"); + + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertEquals(rejected, client.token); + } + + @Test + public void testRefreshAfter401_generationFailureLogsOnlyHttpCodeAndRequestId() { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = rejected; + Map> headers = new HashMap<>(); + headers.put(BaseConstants.REQUEST_ID_HEADER_KEY, java.util.Collections.singletonList("req-123")); + String secretMessage = "invalid credentials file /secrets/sa.json"; + String body = "{\"error\":{\"grpc_code\":16,\"http_code\":401,\"http_status\":\"Unauthorized\"," + + "\"message\":\"" + secretMessage + "\"}}"; + client.failure = new SkyflowException(401, new RuntimeException("token endpoint failed"), headers, body); + + LogUtil.setupLogger(LogLevel.WARN); + // setupLogger resets every handler, so attach the capture after it. + Logger logger = Logger.getLogger(LogUtil.class.getName()); + java.util.List records = new ArrayList<>(); + Handler capture = new Handler() { + @Override public void publish(LogRecord record) { records.add(record); } + @Override public void flush() {} + @Override public void close() {} + }; + capture.setLevel(Level.ALL); + logger.addHandler(capture); + try { + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(rejected)); + } finally { + logger.removeHandler(capture); + LogUtil.setupLogger(LogLevel.ERROR); + } + + String warning = records.stream() + .filter(r -> Level.WARNING.equals(r.getLevel())) + .map(LogRecord::getMessage) + .filter(m -> m.contains(WarningLogs.BEARER_TOKEN_REGENERATION_AFTER_401_FAILED.getLog())) + .findFirst() + .orElseThrow(() -> new AssertionError("regeneration-failure warning was not logged")); + Assert.assertTrue(warning, warning.contains("httpCode=401")); + Assert.assertTrue(warning, warning.contains("requestId=req-123")); + Assert.assertFalse("exception message must not be logged: " + warning, warning.contains(secretMessage)); + Assert.assertFalse("response body must not be logged: " + warning, warning.contains("grpc_code")); + Assert.assertFalse("cause must not be logged: " + warning, warning.contains("token endpoint failed")); + } + + @Test + public void testRefreshAfter401_sameTokenBackMeansStop() { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = rejected; + client.script.add(rejected); + + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertEquals(1, client.generations.get()); + } + + @Test + public void testRefreshAfter401_failedGenerationIsNotRepeatedForTheSameToken() { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = rejected; + client.failure = new SkyflowException("auth service down"); + + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertEquals("later 401s on the same token must not generate again", 1, client.generations.get()); + Assert.assertEquals(rejected, client.token); + } + + @Test + public void testRefreshAfter401_sameTokenBackIsNotRepeatedForTheSameToken() { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = rejected; + client.script.add(rejected); + + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertEquals(1, client.generations.get()); + } + + @Test + public void testRefreshAfter401_recoversAfterStartOfMethodCheckStoresNewToken() throws Exception { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = rejected; + client.failure = new SkyflowException("auth service down"); + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(rejected)); + + // Auth comes back; the next method call's start-of-method check mints and stores a token. + client.failure = null; + String stored = jwtExpiringIn(-3); + client.script.add(stored); + client.setBearerToken(null); + Assert.assertEquals(stored, client.token); + + // That token expires too and is rejected: refresh must generate again, not stay stuck. + client.script.add("fresh-after-recovery"); + Assert.assertEquals("fresh-after-recovery", client.refreshBearerTokenAfterUnauthorized(stored)); + Assert.assertEquals(3, client.generations.get()); + } + + @Test + public void testRefreshAfter401_concurrentGenerationFailuresShareOneAttempt() throws Exception { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = rejected; + client.failure = new SkyflowException("auth service down"); + int threads = 8; + ExecutorService pool = Executors.newFixedThreadPool(threads); + CountDownLatch start = new CountDownLatch(1); + java.util.List> results = new ArrayList<>(); + try { + for (int i = 0; i < threads; i++) { + results.add(pool.submit(() -> { + start.await(); + return client.refreshBearerTokenAfterUnauthorized(rejected); + })); + } + start.countDown(); + for (Future result : results) { + Assert.assertNull(result.get(10, TimeUnit.SECONDS)); + } + } finally { + pool.shutdownNow(); + } + Assert.assertEquals("eight 401s during an auth outage must cost one failed mint, not eight", 1, client.generations.get()); + Assert.assertEquals(rejected, client.token); + } + + @Test + public void testRefreshAfter401_reusesTokenAnotherRequestAlreadyRegenerated() { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = "already-fresh"; + client.script.add("never-used"); + + Assert.assertEquals("already-fresh", client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertEquals(0, client.generations.get()); + } + + @Test + public void testRefreshAfter401_concurrentRejectionsShareOneGeneration() throws Exception { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = rejected; + client.script.add("fresh-token"); + int threads = 8; + ExecutorService pool = Executors.newFixedThreadPool(threads); + CountDownLatch start = new CountDownLatch(1); + java.util.List> results = new ArrayList<>(); + try { + for (int i = 0; i < threads; i++) { + results.add(pool.submit(() -> { + start.await(); + return client.refreshBearerTokenAfterUnauthorized(rejected); + })); + } + start.countDown(); + for (Future result : results) { + Assert.assertEquals("fresh-token", result.get(10, TimeUnit.SECONDS)); + } + } finally { + pool.shutdownNow(); + } + Assert.assertEquals("eight 401s on one token must cost one mint", 1, client.generations.get()); + } + + /** Fake chain that replays scripted status codes and records what was proceeded. */ + private static final class ScriptedChain implements Interceptor.Chain { + private final Request request = new Request.Builder().url("https://example.com/v1/x").build(); + private final int[] codes; + final java.util.List proceeded = new ArrayList<>(); + + ScriptedChain(int... codes) { + this.codes = codes; + } + + @Override + public Request request() { + return request; + } + + @Override + public Response proceed(Request request) { + proceeded.add(request); + int code = codes[Math.min(proceeded.size() - 1, codes.length - 1)]; + return new Response.Builder().request(request).protocol(Protocol.HTTP_1_1) + .code(code).message("status " + code) + .body(okhttp3.ResponseBody.create(null, new byte[0])) + .build(); + } + + @Override + public Connection connection() { + return null; + } + + @Override + public Call call() { + throw new UnsupportedOperationException(); + } + + @Override + public int connectTimeoutMillis() { + return 0; + } + + @Override + public Interceptor.Chain withConnectTimeout(int timeout, TimeUnit unit) { + return this; + } + + @Override + public int readTimeoutMillis() { + return 0; + } + + @Override + public Interceptor.Chain withReadTimeout(int timeout, TimeUnit unit) { + return this; + } + + @Override + public int writeTimeoutMillis() { + return 0; + } + + @Override + public Interceptor.Chain withWriteTimeout(int timeout, TimeUnit unit) { + return this; + } + } + + @Test + public void testAuthInterceptor_401OnExpiredTokenIsResentOnceWithFreshToken() throws IOException { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + client.token = jwtExpiringIn(-2); + client.script.add("fresh-token"); + ScriptedChain chain = new ScriptedChain(401, 200); + + Response response = client.authInterceptor(() -> client.token).intercept(chain); + + Assert.assertEquals(200, response.code()); + Assert.assertEquals(2, chain.proceeded.size()); + Assert.assertEquals("Bearer fresh-token", chain.proceeded.get(1).header("Authorization")); + Assert.assertEquals(chain.request().url(), chain.proceeded.get(1).url()); + } + + @Test + public void testAuthInterceptor_second401IsReturnedNotRetried() throws IOException { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + client.token = jwtExpiringIn(-2); + client.script.add("fresh-token"); + ScriptedChain chain = new ScriptedChain(401, 401); + + Assert.assertEquals(401, client.authInterceptor(() -> client.token).intercept(chain).code()); + Assert.assertEquals(2, chain.proceeded.size()); + Assert.assertEquals(1, client.generations.get()); + } + + @Test + public void testAuthInterceptor_401OnStaticTokenIsNotRetried() throws IOException { + BaseCredentials credentials = new BaseCredentials(); + credentials.setToken(jwtExpiringIn(-2)); + ScriptedClient client = new ScriptedClient(credentials); + client.token = credentials.getToken(); + ScriptedChain chain = new ScriptedChain(401, 200); + + Assert.assertEquals(401, client.authInterceptor(() -> client.token).intercept(chain).code()); + Assert.assertEquals(1, chain.proceeded.size()); + } + + @Test + public void testAuthInterceptor_401OnLiveTokenIsNotRetried() throws IOException { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + client.token = jwtExpiringIn(600); + client.script.add("fresh-token"); + ScriptedChain chain = new ScriptedChain(401, 200); + + Assert.assertEquals(401, client.authInterceptor(() -> client.token).intercept(chain).code()); + Assert.assertEquals(1, chain.proceeded.size()); + Assert.assertEquals(0, client.generations.get()); + } + + @Test + public void testAuthInterceptor_403IsNotRetried() throws IOException { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + client.token = jwtExpiringIn(-2); + client.script.add("fresh-token"); + ScriptedChain chain = new ScriptedChain(403, 200); + + Assert.assertEquals(403, client.authInterceptor(() -> client.token).intercept(chain).code()); + Assert.assertEquals(1, chain.proceeded.size()); + Assert.assertEquals(0, client.generations.get()); + } + + // ── gaps found by line coverage of the fix ─────────────────────────────── + + @Test + public void testAuthInterceptor_readsTokenThroughGetTokenOnEveryAttempt() throws IOException { + // The SDKs hand the interceptor this::getToken, not a captured value: the resend must see the + // token the refresh just stored, and the synchronized getter is what makes that visible. + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = rejected; + client.script.add("fresh-token"); + ScriptedChain chain = new ScriptedChain(401, 200); + + Response response = client.authInterceptor(client::getToken).intercept(chain); + + Assert.assertEquals(200, response.code()); + Assert.assertEquals("Bearer " + rejected, chain.proceeded.get(0).header("Authorization")); + Assert.assertEquals("Bearer fresh-token", chain.proceeded.get(1).header("Authorization")); + Assert.assertEquals("fresh-token", client.getToken()); + } + + @Test + public void testRefreshAfter401_nullFromGenerationIsAFailureAndIsRemembered() { + // BearerToken.getBearerToken() returns null when the credentials match neither FILE nor STRING. + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = rejected; + client.script.add(null); + + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertEquals("a null token must never be cached", rejected, client.token); + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertEquals("the failed attempt is remembered for this token", 1, client.generations.get()); + } + + @Test + public void testRefreshAfter401_cacheClearedWhileRequestInFlightRegenerates() { + // prioritiseCredentials nulls the cache on a credential change. A 401 for the token that was + // already in flight must regenerate rather than treat the empty cache as "someone else refreshed". + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = null; + client.script.add("fresh-token"); + + Assert.assertEquals("fresh-token", client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertEquals("fresh-token", client.token); + Assert.assertEquals(1, client.generations.get()); + } + + @Test + public void testRefreshAfter401_credentialChangeForgetsFailedRegeneration() throws SkyflowException { + ScriptedClient client = new ScriptedClient(mintableCredentials()); + String rejected = jwtExpiringIn(-2); + client.token = rejected; + client.failure = new SkyflowException("auth service down"); + Assert.assertNull(client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertEquals(1, client.generations.get()); + + // New credentials: the failure recorded under the old ones must not block the first 401 on them. + client.prioritiseCredentials(mintableCredentials()); + client.failure = null; + client.token = rejected; // the expired token is still what the in-flight request sent + client.script.add("fresh-token"); + + Assert.assertEquals("fresh-token", client.refreshBearerTokenAfterUnauthorized(rejected)); + Assert.assertEquals("fresh-token", client.token); + Assert.assertEquals(2, client.generations.get()); + } } diff --git a/common/src/test/java/com/skyflow/detect/TestDeidentifySource.java b/common/src/test/java/com/skyflow/detect/TestDeidentifySource.java new file mode 100644 index 00000000..471356fc --- /dev/null +++ b/common/src/test/java/com/skyflow/detect/TestDeidentifySource.java @@ -0,0 +1,16 @@ +package com.skyflow.detect; + +/** Test stand-in for the per-SDK {@code DeidentifySourceType} enums, which common cannot see. Carries every API value. */ +public enum TestDeidentifySource implements DeidentifyFileSource { + BASE64, + SKYFLOW_ID, + PRESIGNED_URL; + + public static TestDeidentifySource fromWire(String wireValue) { + try { + return wireValue == null ? null : valueOf(wireValue); + } catch (IllegalArgumentException unknown) { + return null; + } + } +} diff --git a/common/src/test/java/com/skyflow/detect/TestRedactionLevel.java b/common/src/test/java/com/skyflow/detect/TestRedactionLevel.java new file mode 100644 index 00000000..5a2cece9 --- /dev/null +++ b/common/src/test/java/com/skyflow/detect/TestRedactionLevel.java @@ -0,0 +1,89 @@ +package com.skyflow.detect; + +import com.skyflow.enums.RedactionType; + +/** Test stand-in for the per-SDK {@code RedactionLevel} classes, which common cannot see. Carries all four API fields. */ +public final class TestRedactionLevel extends RedactionRule { + private final EntityType entityName; + private final RedactionType redactionType; + private final String tokenGroupName; + private final String redactionPattern; + + private TestRedactionLevel(Builder b) { + this.entityName = b.entityName; + this.redactionType = b.redactionType; + this.tokenGroupName = b.tokenGroupName; + this.redactionPattern = b.redactionPattern; + } + + public static Builder builder() { + return new Builder(); + } + + public EntityType getEntityName() { + return entityName; + } + + public RedactionType getRedactionType() { + return redactionType; + } + + public String getTokenGroupName() { + return tokenGroupName; + } + + public String getRedactionPattern() { + return redactionPattern; + } + + @Override + String apiEntityName() { + return entityName == null ? null : entityName.name(); + } + + @Override + String apiRedactionType() { + return redactionType == null ? null : redactionType.name(); + } + + @Override + String apiTokenGroupName() { + return tokenGroupName; + } + + @Override + String apiRedactionPattern() { + return redactionPattern; + } + + public static final class Builder { + private EntityType entityName; + private RedactionType redactionType; + private String tokenGroupName; + private String redactionPattern; + + public Builder entityName(EntityType entityName) { + this.entityName = entityName; + return this; + } + + public Builder redactionType(RedactionType redactionType) { + this.redactionType = redactionType; + return this; + } + + public Builder tokenGroupName(String tokenGroupName) { + this.tokenGroupName = tokenGroupName; + return this; + } + + public Builder redactionPattern(String redactionPattern) { + this.redactionPattern = redactionPattern; + return this; + } + + public TestRedactionLevel build() { + return new TestRedactionLevel(this); + } + } +} diff --git a/common/src/test/java/com/skyflow/detect/TestReidentifySource.java b/common/src/test/java/com/skyflow/detect/TestReidentifySource.java new file mode 100644 index 00000000..bba36ebc --- /dev/null +++ b/common/src/test/java/com/skyflow/detect/TestReidentifySource.java @@ -0,0 +1,15 @@ +package com.skyflow.detect; + +/** Test stand-in for the per-SDK {@code ReidentifySourceType} enums, which common cannot see. */ +public enum TestReidentifySource implements ReidentifyFileSource { + BASE64, + PRESIGNED_URL; + + public static TestReidentifySource fromWire(String wireValue) { + try { + return wireValue == null ? null : valueOf(wireValue); + } catch (IllegalArgumentException unknown) { + return null; + } + } +} diff --git a/common/src/test/java/com/skyflow/detect/internal/DeidentifyFileFlowTests.java b/common/src/test/java/com/skyflow/detect/internal/DeidentifyFileFlowTests.java index a55d5078..fa3008da 100644 --- a/common/src/test/java/com/skyflow/detect/internal/DeidentifyFileFlowTests.java +++ b/common/src/test/java/com/skyflow/detect/internal/DeidentifyFileFlowTests.java @@ -1,6 +1,6 @@ package com.skyflow.detect.internal; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.TestDeidentifySource; import com.skyflow.detect.DeidentifyFileRequest; import com.skyflow.detect.DeidentifyFileResponse; import com.skyflow.detect.DetectRunStatus; @@ -30,10 +30,10 @@ public class DeidentifyFileFlowTests { private static final RunPoller.Sleeper NO_SLEEP = millis -> { }; private static GetRunResponse success() { - return success(DataSourceType.BASE64); + return success(TestDeidentifySource.BASE64); } - private static GetRunResponse success(DataSourceType outputType) { + private static GetRunResponse success(TestDeidentifySource outputType) { return new GetRunResponse("run-1", DetectRunStatus.SUCCESS, outputType, Arrays.asList( new FileOutput(Base64.getEncoder().encodeToString("redacted".getBytes()), @@ -47,7 +47,7 @@ private static GetRunResponse success(DataSourceType outputType) { public void withoutPollOptionsReturnsRunIdAndNeverFetches() throws Exception { AtomicInteger fetches = new AtomicInteger(); DeidentifyFileResponse response = DeidentifyFileFlow.complete( - DeidentifyFileRequest.builder().dataSource(DataSourceType.BASE64).value("x").build(), "run-1", + DeidentifyFileRequest.builder().dataSource(TestDeidentifySource.BASE64).value("x").build(), "run-1", id -> { fetches.incrementAndGet(); return success(); }, NO_SLEEP); Assert.assertEquals("run-1", response.getRunId()); Assert.assertNull(response.getStatus()); @@ -59,7 +59,7 @@ public void withoutPollOptionsReturnsRunIdAndNeverFetches() throws Exception { public void missingRunIdSkipsPollingEvenWhenRequested() throws Exception { AtomicInteger fetches = new AtomicInteger(); DeidentifyFileResponse response = DeidentifyFileFlow.complete( - DeidentifyFileRequest.builder().dataSource(DataSourceType.BASE64).value("x") + DeidentifyFileRequest.builder().dataSource(TestDeidentifySource.BASE64).value("x") .pollOptions(PollOptions.builder().build()).build(), null, id -> { fetches.incrementAndGet(); return success(); }, NO_SLEEP); Assert.assertNull(response.getRunId()); @@ -70,7 +70,7 @@ public void missingRunIdSkipsPollingEvenWhenRequested() throws Exception { @Test public void polledRawRequestMergesRunButWritesNothing() throws Exception { DeidentifyFileResponse response = DeidentifyFileFlow.complete( - DeidentifyFileRequest.builder().dataSource(DataSourceType.SKYFLOW_ID).value("rec") + DeidentifyFileRequest.builder().dataSource(TestDeidentifySource.SKYFLOW_ID).value("rec") .pollOptions(PollOptions.builder().build()).outputDirectory(folder.getRoot().getPath()).build(), "run-1", id -> success(), NO_SLEEP); Assert.assertEquals(DetectRunStatus.SUCCESS, response.getStatus()); @@ -108,10 +108,10 @@ public void polledFileRequestWritesNothingForPresignedUrlOutputs() throws Except DeidentifyFileResponse response = DeidentifyFileFlow.complete( DeidentifyFileRequest.builder().file(input).pollOptions(PollOptions.builder().build()) .outputDirectory(outDir.getPath()).build(), - "run-1", id -> success(DataSourceType.PRESIGNED_URL), NO_SLEEP); + "run-1", id -> success(TestDeidentifySource.PRESIGNED_URL), NO_SLEEP); Assert.assertEquals(DetectRunStatus.SUCCESS, response.getStatus()); - Assert.assertEquals(DataSourceType.PRESIGNED_URL, response.getOutputType()); + Assert.assertEquals(TestDeidentifySource.PRESIGNED_URL, response.getOutputType()); Assert.assertEquals(2, response.getOutput().size()); Assert.assertEquals(0, outDir.list().length); } diff --git a/common/src/test/java/com/skyflow/detect/internal/DeidentifyFileMapperTests.java b/common/src/test/java/com/skyflow/detect/internal/DeidentifyFileMapperTests.java index d6fa068e..90866595 100644 --- a/common/src/test/java/com/skyflow/detect/internal/DeidentifyFileMapperTests.java +++ b/common/src/test/java/com/skyflow/detect/internal/DeidentifyFileMapperTests.java @@ -1,6 +1,6 @@ package com.skyflow.detect.internal; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.TestDeidentifySource; import com.skyflow.detect.DeidentifyFileRequest; import com.skyflow.detect.DeidentifyFileResponse; import com.skyflow.detect.Detect; @@ -37,7 +37,7 @@ private static String wire(DeidentifyFileRequestV2 request) throws Exception { @Test public void mapsRawValueRequest() throws Exception { DeidentifyFileRequestV2 api = DetectRequestMapper.toDeidentifyFileRequest(DeidentifyFileRequest.builder() - .dataSource(DataSourceType.SKYFLOW_ID).value("rec-1").configurationId("cfg-1").build(), "vault-1"); + .dataSource(TestDeidentifySource.SKYFLOW_ID).value("rec-1").configurationId("cfg-1").build(), "vault-1"); Assert.assertEquals("SKYFLOW_ID", api.getDataSource().toString()); Assert.assertEquals("rec-1", api.getValue()); Assert.assertEquals("cfg-1", api.getConfigurationId().get()); @@ -53,14 +53,14 @@ public void mapsRawValueRequest() throws Exception { @Test public void dataFormatIsSentLowerCase() throws Exception { DeidentifyFileRequestV2 api = DetectRequestMapper.toDeidentifyFileRequest(DeidentifyFileRequest.builder() - .dataSource(DataSourceType.PRESIGNED_URL).value("https://s3/in").dataFormat(FileDataFormat.PDF).build(), "v"); + .dataSource(TestDeidentifySource.PRESIGNED_URL).value("https://s3/in").dataFormat(FileDataFormat.PDF).build(), "v"); Assert.assertTrue(wire(api).contains("\"dataFormat\":\"pdf\"")); } @Test public void noConfigurationSendsVaultIdOnlyConfiguration() throws Exception { DeidentifyFileRequestV2 api = DetectRequestMapper.toDeidentifyFileRequest(DeidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64).value("aGVsbG8=").build(), "vault-1"); + .dataSource(TestDeidentifySource.BASE64).value("aGVsbG8=").build(), "vault-1"); Assert.assertFalse(api.getConfigurationId().isPresent()); Assert.assertEquals("vault-1", api.getConfiguration().get().getVaultId()); Assert.assertFalse(api.getConfiguration().get().getDetect().isPresent()); @@ -71,7 +71,7 @@ public void noConfigurationSendsVaultIdOnlyConfiguration() throws Exception { @Test public void storedConfigurationIdSendsNoConfiguration() throws Exception { DeidentifyFileRequestV2 api = DetectRequestMapper.toDeidentifyFileRequest(DeidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64).value("aGVsbG8=").configurationId("cfg-1").build(), "vault-1"); + .dataSource(TestDeidentifySource.BASE64).value("aGVsbG8=").configurationId("cfg-1").build(), "vault-1"); Assert.assertEquals("cfg-1", api.getConfigurationId().get()); Assert.assertFalse(api.getConfiguration().isPresent()); } @@ -79,7 +79,7 @@ public void storedConfigurationIdSendsNoConfiguration() throws Exception { @Test public void inlineConfigurationGetsVaultId() throws Exception { DeidentifyFileRequestV2 api = DetectRequestMapper.toDeidentifyFileRequest(DeidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64).value("Zm9v") + .dataSource(TestDeidentifySource.BASE64).value("Zm9v") .configuration(DetectConfiguration.builder().detect(Detect.builder().entities(Collections.singletonList( Entity.builder().entityType(EntityType.NAME).build())).build()).build()) .build(), "vault-9"); @@ -153,11 +153,11 @@ public void polledResponseMergesRunIdWithRun() throws Exception { "{\"status\":\"SUCCESS\",\"outputType\":\"BASE64\",\"output\":[{\"processedFile\":\"UERG\"," + "\"processedFileType\":\"REDACTED_FILE\",\"processedFileExtension\":\"pdf\"}],\"metrics\":{\"pages\":2}}", DetectRunsResponseV2.class); - GetRunResponse run = DetectResponseMapper.toGetRunResponse("run-7", api); + GetRunResponse run = DetectResponseMapper.toGetRunResponse("run-7", api, TestDeidentifySource::fromWire); DeidentifyFileResponse response = DetectResponseMapper.toDeidentifyFileResponse("run-7", run); Assert.assertEquals("run-7", response.getRunId()); Assert.assertEquals(DetectRunStatus.SUCCESS, response.getStatus()); - Assert.assertEquals(DataSourceType.BASE64, response.getOutputType()); + Assert.assertEquals(TestDeidentifySource.BASE64, response.getOutputType()); Assert.assertEquals("UERG", response.getOutput().get(0).getProcessedFile()); Assert.assertEquals(FileDataFormat.PDF, response.getOutput().get(0).getProcessedFileExtension()); Assert.assertEquals(Integer.valueOf(2), response.getMetrics().getPages()); diff --git a/common/src/test/java/com/skyflow/detect/internal/DeidentifyFileValidationsTests.java b/common/src/test/java/com/skyflow/detect/internal/DeidentifyFileValidationsTests.java index 9f9a3f62..99567356 100644 --- a/common/src/test/java/com/skyflow/detect/internal/DeidentifyFileValidationsTests.java +++ b/common/src/test/java/com/skyflow/detect/internal/DeidentifyFileValidationsTests.java @@ -1,6 +1,7 @@ package com.skyflow.detect.internal; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.TestDeidentifySource; + import com.skyflow.detect.DeidentifyFileRequest; import com.skyflow.detect.PollOptions; import com.skyflow.errors.BaseErrorMessage; @@ -30,7 +31,7 @@ private static void assertRejected(DeidentifyFileRequest request, BaseErrorMessa } private static DeidentifyFileRequest.DeidentifyFileRequestBuilder raw() { - return DeidentifyFileRequest.builder().dataSource(DataSourceType.BASE64).value("Zm9v"); + return DeidentifyFileRequest.builder().dataSource(TestDeidentifySource.BASE64).value("Zm9v"); } @Test @@ -41,7 +42,7 @@ public void rejectsNullRequest() { @Test public void rawValueRequiresDataSourceAndValue() throws SkyflowException { assertRejected(DeidentifyFileRequest.builder().value("x").build(), BaseErrorMessage.InvalidDataSourceInDeidentifyFile); - assertRejected(DeidentifyFileRequest.builder().dataSource(DataSourceType.SKYFLOW_ID).build(), BaseErrorMessage.InvalidValueInDeidentifyFile); + assertRejected(raw().value("").build(), BaseErrorMessage.InvalidValueInDeidentifyFile); assertRejected(raw().value(" ").build(), BaseErrorMessage.InvalidValueInDeidentifyFile); DetectValidations.validateDeidentifyFileRequest(raw().build()); DetectValidations.validateDeidentifyFileRequest(raw().configurationId("cfg").build()); @@ -60,6 +61,25 @@ public void fileRequestsNeedAnExistingFileAndNoValue() throws Exception { assertRejected(DeidentifyFileRequest.builder().file(folder.getRoot()).build(), BaseErrorMessage.FileNotFoundToDeidentify); } + @Test + public void exactlyOneOfFilePathFileOrValueIsRequired() throws Exception { + File existing = folder.newFile("one.txt"); + Files.write(existing.toPath(), "hello".getBytes("UTF-8")); + String path = existing.getPath(); + + assertRejected(DeidentifyFileRequest.builder().build(), BaseErrorMessage.MissingFileSourceInDeidentifyFile); + assertRejected(DeidentifyFileRequest.builder().dataSource(TestDeidentifySource.SKYFLOW_ID).build(), BaseErrorMessage.MissingFileSourceInDeidentifyFile); + + assertRejected(DeidentifyFileRequest.builder().file(existing).filePath(path).build(), BaseErrorMessage.MultipleFileSourcesInDeidentifyFile); + assertRejected(DeidentifyFileRequest.builder().filePath(path).value("Zm9v").build(), BaseErrorMessage.MultipleFileSourcesInDeidentifyFile); + assertRejected(DeidentifyFileRequest.builder().file(existing).value("").build(), BaseErrorMessage.MultipleFileSourcesInDeidentifyFile); + assertRejected(DeidentifyFileRequest.builder().file(existing).filePath(path).value("Zm9v").build(), BaseErrorMessage.MultipleFileSourcesInDeidentifyFile); + + // dataSource alongside a local file is tolerated: the file is always sent as BASE64 + DetectValidations.validateDeidentifyFileRequest(DeidentifyFileRequest.builder().filePath(path).dataSource(TestDeidentifySource.PRESIGNED_URL).build()); + assertRejected(DeidentifyFileRequest.builder().filePath(" ").build(), BaseErrorMessage.FileNotFoundToDeidentify); + } + @Test public void pollOptionsBoundsAreEnforced() throws SkyflowException { DetectValidations.validateDeidentifyFileRequest(raw().pollOptions(PollOptions.builder().build()).build()); diff --git a/common/src/test/java/com/skyflow/detect/internal/DetectRequestMapperTests.java b/common/src/test/java/com/skyflow/detect/internal/DetectRequestMapperTests.java index 8786e0d5..f7c5db54 100644 --- a/common/src/test/java/com/skyflow/detect/internal/DetectRequestMapperTests.java +++ b/common/src/test/java/com/skyflow/detect/internal/DetectRequestMapperTests.java @@ -11,7 +11,7 @@ import com.skyflow.detect.Image; import com.skyflow.detect.MaskingMethod; import com.skyflow.detect.Media; -import com.skyflow.detect.RedactionLevel; +import com.skyflow.detect.TestRedactionLevel; import com.skyflow.enums.RedactionType; import com.skyflow.detect.ReidentifyStringRequest; import com.skyflow.detect.ReturnEntitiesType; @@ -117,8 +117,8 @@ public void reidentifyStringCarriesVaultIdAndRedactionLevels() throws Exception ReidentifyStringRequest.builder() .text("[NAME_1]") .redactionLevel(Arrays.asList( - RedactionLevel.builder().entityName(EntityType.NAME).redactionType(RedactionType.MASKED).build(), - RedactionLevel.builder().tokenGroupName("group").redactionPattern("pattern").build())) + TestRedactionLevel.builder().entityName(EntityType.NAME).redactionType(RedactionType.MASKED).build(), + TestRedactionLevel.builder().tokenGroupName("group").redactionPattern("pattern").build())) .build(), "vault-1"); Assert.assertEquals("vault-1", mapped.getVaultId()); Assert.assertEquals("[NAME_1]", mapped.getText()); @@ -143,7 +143,7 @@ public void everyPublicEntityTypeMapsToAGeneratedValue() { for (EntityType type : EntityType.values()) { Entity entity = Entity.builder().entityType(type).build(); Assert.assertEquals(type.name(), DetectRequestMapper.toEntity(entity).getEntityType().get().toString()); - RedactionLevel level = RedactionLevel.builder().entityName(type).redactionType(RedactionType.DEFAULT).build(); + TestRedactionLevel level = TestRedactionLevel.builder().entityName(type).redactionType(RedactionType.DEFAULT).build(); Assert.assertEquals(type.name(), DetectRequestMapper.toRedactionLevels(Collections.singletonList(level)).get(0).getEntityName().get().toString()); } diff --git a/common/src/test/java/com/skyflow/detect/internal/DetectResponseMapperTests.java b/common/src/test/java/com/skyflow/detect/internal/DetectResponseMapperTests.java index f0ebcada..8909509c 100644 --- a/common/src/test/java/com/skyflow/detect/internal/DetectResponseMapperTests.java +++ b/common/src/test/java/com/skyflow/detect/internal/DetectResponseMapperTests.java @@ -1,7 +1,7 @@ package com.skyflow.detect.internal; import com.skyflow.detect.CheckGuardrailsResponse; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.TestDeidentifySource; import com.skyflow.detect.DeidentifyStringResponse; import com.skyflow.detect.DetectRunStatus; import com.skyflow.detect.FileDataFormat; @@ -61,7 +61,7 @@ public void nullListElementsFromTheApiAreSkipped() throws Exception { DetectRunsResponseV2 run = ObjectMappers.JSON_MAPPER.readValue( "{\"status\":\"SUCCESS\",\"output\":[null,{\"processedFile\":\"x\"}]}", DetectRunsResponseV2.class); - Assert.assertEquals(1, DetectResponseMapper.toGetRunResponse("r", run).getOutput().size()); + Assert.assertEquals(1, DetectResponseMapper.toGetRunResponse("r", run, TestDeidentifySource::fromWire).getOutput().size()); } @Test @@ -99,10 +99,10 @@ public void mapsCompletedRun() throws Exception { "{\"status\":\"SUCCESS\",\"outputType\":\"PRESIGNED_URL\",\"output\":[{\"processedFile\":\"https://x/y.pdf\"," + "\"processedFileType\":\"REDACTED_FILE\",\"processedFileExtension\":\"pdf\"}],\"metrics\":{\"pages\":2}}", DetectRunsResponseV2.class); - GetRunResponse run = DetectResponseMapper.toGetRunResponse("run-1", api); + GetRunResponse run = DetectResponseMapper.toGetRunResponse("run-1", api, TestDeidentifySource::fromWire); Assert.assertEquals("run-1", run.getRunId()); Assert.assertEquals(DetectRunStatus.SUCCESS, run.getStatus()); - Assert.assertEquals(DataSourceType.PRESIGNED_URL, run.getOutputType()); + Assert.assertEquals(TestDeidentifySource.PRESIGNED_URL, run.getOutputType()); Assert.assertEquals("https://x/y.pdf", run.getOutput().get(0).getProcessedFile()); Assert.assertEquals(FileDataFormat.PDF, run.getOutput().get(0).getProcessedFileExtension()); Assert.assertEquals(Integer.valueOf(2), run.getMetrics().getPages()); @@ -111,14 +111,14 @@ public void mapsCompletedRun() throws Exception { @Test public void runWithoutStatusIsUnknownAndUnknownWireValuesAreTolerated() throws Exception { GetRunResponse empty = DetectResponseMapper.toGetRunResponse("r", - ObjectMappers.JSON_MAPPER.readValue("{}", DetectRunsResponseV2.class)); + ObjectMappers.JSON_MAPPER.readValue("{}", DetectRunsResponseV2.class), TestDeidentifySource::fromWire); Assert.assertEquals(DetectRunStatus.UNKNOWN, empty.getStatus()); Assert.assertTrue(empty.getOutput().isEmpty()); Assert.assertNull(empty.getOutputType()); GetRunResponse odd = DetectResponseMapper.toGetRunResponse("r", ObjectMappers.JSON_MAPPER.readValue( "{\"status\":\"NEW_STATE\",\"outputType\":\"FTP\",\"output\":[{\"processedFileExtension\":\"heic\"}]}", - DetectRunsResponseV2.class)); + DetectRunsResponseV2.class), TestDeidentifySource::fromWire); Assert.assertEquals(DetectRunStatus.UNKNOWN, odd.getStatus()); Assert.assertNull(odd.getOutputType()); Assert.assertNull(odd.getOutput().get(0).getProcessedFileExtension()); diff --git a/common/src/test/java/com/skyflow/detect/internal/DetectValidationsTests.java b/common/src/test/java/com/skyflow/detect/internal/DetectValidationsTests.java index b663f805..09fab254 100644 --- a/common/src/test/java/com/skyflow/detect/internal/DetectValidationsTests.java +++ b/common/src/test/java/com/skyflow/detect/internal/DetectValidationsTests.java @@ -5,7 +5,7 @@ import com.skyflow.detect.DetectConfiguration; import com.skyflow.detect.EntityType; import com.skyflow.detect.GetRunRequest; -import com.skyflow.detect.RedactionLevel; +import com.skyflow.detect.TestRedactionLevel; import com.skyflow.enums.RedactionType; import com.skyflow.detect.ReidentifyStringRequest; import com.skyflow.errors.BaseErrorMessage; @@ -79,20 +79,20 @@ public void reidentifyStringAcceptsValidRedactionLevels() throws SkyflowExceptio DetectValidations.validateReidentifyStringRequest(ReidentifyStringRequest.builder() .text("[NAME_1]") .redactionLevel(Arrays.asList( - RedactionLevel.builder().entityName(EntityType.NAME).redactionType(RedactionType.MASKED).build(), - RedactionLevel.builder().tokenGroupName("group").redactionPattern("pattern").build())) + TestRedactionLevel.builder().entityName(EntityType.NAME).redactionType(RedactionType.MASKED).build(), + TestRedactionLevel.builder().tokenGroupName("group").redactionPattern("pattern").build())) .build()); DetectValidations.validateReidentifyStringRequest(ReidentifyStringRequest.builder().text("[NAME_1]").build()); } @Test public void reidentifyStringPassesAnyRedactionLevelShapeThrough() throws SkyflowException { - // The API marks every RedactionLevel field optional; the server applies its own rules. + // The API marks every TestRedactionLevel field optional; the server applies its own rules. DetectValidations.validateReidentifyStringRequest(ReidentifyStringRequest.builder().text("[NAME_1]") .redactionLevel(Arrays.asList( - RedactionLevel.builder().build(), - RedactionLevel.builder().tokenGroupName("g").build(), - RedactionLevel.builder().entityName(EntityType.NAME).redactionType(RedactionType.MASKED).redactionPattern("p").build(), + TestRedactionLevel.builder().build(), + TestRedactionLevel.builder().tokenGroupName("g").build(), + TestRedactionLevel.builder().entityName(EntityType.NAME).redactionType(RedactionType.MASKED).redactionPattern("p").build(), null)) .build()); } diff --git a/common/src/test/java/com/skyflow/detect/internal/FileOutputWriterTests.java b/common/src/test/java/com/skyflow/detect/internal/FileOutputWriterTests.java index 70297c59..a955266c 100644 --- a/common/src/test/java/com/skyflow/detect/internal/FileOutputWriterTests.java +++ b/common/src/test/java/com/skyflow/detect/internal/FileOutputWriterTests.java @@ -1,6 +1,6 @@ package com.skyflow.detect.internal; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.TestDeidentifySource; import com.skyflow.detect.FileDataFormat; import com.skyflow.detect.FileOutput; import com.skyflow.errors.SkyflowException; @@ -30,7 +30,7 @@ public void writesEachOutputWithV1StyleNames() throws Exception { List written = FileOutputWriter.write(Arrays.asList( new FileOutput(b64("%PDF"), "REDACTED_FILE", FileDataFormat.PDF), new FileOutput(b64("[]"), "ENTITIES", null)), - DataSourceType.BASE64, "invoice.pdf", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "invoice.pdf", folder.getRoot().getPath()); Assert.assertEquals(2, written.size()); Assert.assertEquals("processed-invoice.pdf", written.get(0).getName()); @@ -45,7 +45,7 @@ public void entityOutputsCarryTheirTypeInTheName() throws Exception { List written = FileOutputWriter.write(Arrays.asList( new FileOutput(b64("a"), "ENTITIES", FileDataFormat.JSON), new FileOutput(b64("b"), "OBJECT_ENTITIES", FileDataFormat.JSON)), - DataSourceType.BASE64, "scan.png", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "scan.png", folder.getRoot().getPath()); Assert.assertEquals("processed-entities-scan.json", written.get(0).getName()); Assert.assertEquals("processed-object-entities-scan.json", written.get(1).getName()); } @@ -55,7 +55,7 @@ public void jsonInputDoesNotCollideWithItsEntitiesOutput() throws Exception { List written = FileOutputWriter.write(Arrays.asList( new FileOutput(b64("{}"), "REDACTED_TEXT", FileDataFormat.JSON), new FileOutput(b64("[]"), "ENTITIES", FileDataFormat.JSON)), - DataSourceType.BASE64, "data.json", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "data.json", folder.getRoot().getPath()); Assert.assertEquals("processed-data.json", written.get(0).getName()); Assert.assertEquals("processed-entities-data.json", written.get(1).getName()); } @@ -65,7 +65,7 @@ public void nameCollisionsGetTheFileTypeInserted() throws Exception { List written = FileOutputWriter.write(Arrays.asList( new FileOutput(b64("a"), "REDACTED_FILE", FileDataFormat.TXT), new FileOutput(b64("b"), "REDACTED_TEXT", FileDataFormat.TXT)), - DataSourceType.BASE64, "notes.txt", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "notes.txt", folder.getRoot().getPath()); Assert.assertEquals("processed-notes.txt", written.get(0).getName()); Assert.assertEquals("processed-notes.redacted_text.txt", written.get(1).getName()); } @@ -75,7 +75,7 @@ public void createsMissingOutputDirectory() throws Exception { File nested = new File(folder.getRoot(), "a/b/c"); List written = FileOutputWriter.write( Collections.singletonList(new FileOutput(b64("x"), "REDACTED_TEXT", FileDataFormat.TXT)), - DataSourceType.BASE64, "notes.txt", nested.getPath()); + TestDeidentifySource.BASE64, "notes.txt", nested.getPath()); Assert.assertTrue(nested.isDirectory()); Assert.assertEquals(new File(nested, "processed-notes.txt"), written.get(0)); } @@ -84,24 +84,24 @@ public void createsMissingOutputDirectory() throws Exception { public void presignedUrlOutputsAreNotWritten() throws Exception { List written = FileOutputWriter.write( Collections.singletonList(new FileOutput("https://s3/out", "REDACTED_FILE", FileDataFormat.PDF)), - DataSourceType.PRESIGNED_URL, "invoice.pdf", folder.getRoot().getPath()); + TestDeidentifySource.PRESIGNED_URL, "invoice.pdf", folder.getRoot().getPath()); Assert.assertTrue(written.isEmpty()); Assert.assertEquals(0, folder.getRoot().list().length); } @Test public void emptyOrNullOutputsWriteNothing() throws Exception { - Assert.assertTrue(FileOutputWriter.write(null, DataSourceType.BASE64, "x.txt", folder.getRoot().getPath()).isEmpty()); - Assert.assertTrue(FileOutputWriter.write(Collections.emptyList(), DataSourceType.BASE64, "x.txt", folder.getRoot().getPath()).isEmpty()); + Assert.assertTrue(FileOutputWriter.write(null, TestDeidentifySource.BASE64, "x.txt", folder.getRoot().getPath()).isEmpty()); + Assert.assertTrue(FileOutputWriter.write(Collections.emptyList(), TestDeidentifySource.BASE64, "x.txt", folder.getRoot().getPath()).isEmpty()); Assert.assertTrue(FileOutputWriter.write(Collections.singletonList(new FileOutput(null, "ENTITIES", null)), - DataSourceType.BASE64, "x.txt", folder.getRoot().getPath()).isEmpty()); + TestDeidentifySource.BASE64, "x.txt", folder.getRoot().getPath()).isEmpty()); } @Test public void invalidBase64IsReportedAsWriteFailure() { try { FileOutputWriter.write(Collections.singletonList(new FileOutput("not base64!", "REDACTED_FILE", FileDataFormat.TXT)), - DataSourceType.BASE64, "x.txt", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "x.txt", folder.getRoot().getPath()); Assert.fail("expected SkyflowException"); } catch (SkyflowException e) { Assert.assertTrue(e.getMessage(), e.getMessage().contains("processed-x.txt")); @@ -115,7 +115,7 @@ public void nullOrEmptyTypeFallsBackToPlainName() throws Exception { List written = FileOutputWriter.write(Arrays.asList( new FileOutput(b64("a"), null, FileDataFormat.PDF), new FileOutput(b64("b"), "", FileDataFormat.TXT)), - DataSourceType.BASE64, "invoice.pdf", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "invoice.pdf", folder.getRoot().getPath()); Assert.assertEquals("processed-invoice.pdf", written.get(0).getName()); Assert.assertEquals("processed-invoice.txt", written.get(1).getName()); } @@ -124,7 +124,7 @@ public void nullOrEmptyTypeFallsBackToPlainName() throws Exception { public void nonEntityOutputWithoutExtensionGetsBin() throws Exception { List written = FileOutputWriter.write( Collections.singletonList(new FileOutput(b64("x"), "REDACTED_TRANSCRIPTION", null)), - DataSourceType.BASE64, "call.mp3", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "call.mp3", folder.getRoot().getPath()); Assert.assertEquals("processed-call.bin", written.get(0).getName()); } @@ -132,7 +132,7 @@ public void nonEntityOutputWithoutExtensionGetsBin() throws Exception { public void entityOutputHonoursAnExplicitExtension() throws Exception { List written = FileOutputWriter.write( Collections.singletonList(new FileOutput(b64("x"), "ENTITIES", FileDataFormat.TXT)), - DataSourceType.BASE64, "notes.txt", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "notes.txt", folder.getRoot().getPath()); Assert.assertEquals("processed-entities-notes.txt", written.get(0).getName()); } @@ -140,7 +140,7 @@ public void entityOutputHonoursAnExplicitExtension() throws Exception { public void entityTypeMatchIsCaseInsensitive() throws Exception { List written = FileOutputWriter.write( Collections.singletonList(new FileOutput(b64("x"), "entities", null)), - DataSourceType.BASE64, "notes.txt", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "notes.txt", folder.getRoot().getPath()); Assert.assertEquals("processed-entities-notes.json", written.get(0).getName()); } @@ -149,7 +149,7 @@ public void duplicateEntityOutputsGetTheTypeInsertedOnCollision() throws Excepti List written = FileOutputWriter.write(Arrays.asList( new FileOutput(b64("a"), "ENTITIES", FileDataFormat.JSON), new FileOutput(b64("b"), "ENTITIES", FileDataFormat.JSON)), - DataSourceType.BASE64, "scan.png", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "scan.png", folder.getRoot().getPath()); Assert.assertEquals("processed-entities-scan.json", written.get(0).getName()); Assert.assertEquals("processed-entities-scan.entities.json", written.get(1).getName()); } @@ -160,7 +160,7 @@ public void collisionWithoutATypeUsesARunningNumber() throws Exception { new FileOutput(b64("a"), null, FileDataFormat.TXT), new FileOutput(b64("b"), null, FileDataFormat.TXT), new FileOutput(b64("c"), null, FileDataFormat.TXT)), - DataSourceType.BASE64, "notes.txt", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "notes.txt", folder.getRoot().getPath()); Assert.assertEquals("processed-notes.txt", written.get(0).getName()); Assert.assertEquals("processed-notes.1.txt", written.get(1).getName()); Assert.assertEquals("processed-notes.2.txt", written.get(2).getName()); @@ -170,12 +170,12 @@ public void collisionWithoutATypeUsesARunningNumber() throws Exception { public void inputNamesWithoutAnExtensionOrWithLeadingDotAreKept() throws Exception { List plain = FileOutputWriter.write( Collections.singletonList(new FileOutput(b64("x"), "ENTITIES", null)), - DataSourceType.BASE64, "README", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "README", folder.getRoot().getPath()); Assert.assertEquals("processed-entities-README.json", plain.get(0).getName()); List dotted = FileOutputWriter.write( Collections.singletonList(new FileOutput(b64("x"), "REDACTED_TEXT", FileDataFormat.TXT)), - DataSourceType.BASE64, ".notes", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, ".notes", folder.getRoot().getPath()); Assert.assertEquals("processed-.notes.txt", dotted.get(0).getName()); } @@ -184,7 +184,7 @@ public void missingInputNameFallsBackToOutput() throws Exception { List written = FileOutputWriter.write(Arrays.asList( new FileOutput(b64("a"), "REDACTED_FILE", FileDataFormat.PDF), new FileOutput(b64("b"), "ENTITIES", null)), - DataSourceType.BASE64, null, folder.getRoot().getPath()); + TestDeidentifySource.BASE64, null, folder.getRoot().getPath()); Assert.assertEquals("processed-output.pdf", written.get(0).getName()); Assert.assertEquals("processed-entities-output.json", written.get(1).getName()); } @@ -205,7 +205,7 @@ public void nullElementsInTheOutputListAreSkipped() throws Exception { List written = FileOutputWriter.write(Arrays.asList( null, new FileOutput(b64("x"), "REDACTED_TEXT", FileDataFormat.TXT)), - DataSourceType.BASE64, "notes.txt", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "notes.txt", folder.getRoot().getPath()); Assert.assertEquals(1, written.size()); Assert.assertEquals("processed-notes.txt", written.get(0).getName()); } @@ -216,7 +216,7 @@ public void blankOutputDirectoryWritesToTheWorkingDirectory() throws Exception { try { List written = FileOutputWriter.write( Collections.singletonList(new FileOutput(b64("x"), "REDACTED_TEXT", FileDataFormat.TXT)), - DataSourceType.BASE64, "writer-cwd-test.txt", " "); + TestDeidentifySource.BASE64, "writer-cwd-test.txt", " "); Assert.assertEquals(expected, written.get(0)); Assert.assertNull(written.get(0).getParentFile()); Assert.assertTrue(expected.isFile()); @@ -231,7 +231,7 @@ public void outputDirectoryThatIsAFileIsReportedAsWriteFailure() throws Exceptio try { FileOutputWriter.write( Collections.singletonList(new FileOutput(b64("x"), "REDACTED_TEXT", FileDataFormat.TXT)), - DataSourceType.BASE64, "notes.txt", notADirectory.getPath()); + TestDeidentifySource.BASE64, "notes.txt", notADirectory.getPath()); Assert.fail("expected SkyflowException"); } catch (SkyflowException e) { Assert.assertTrue(e.getMessage(), e.getMessage().contains(notADirectory.getPath())); @@ -244,7 +244,7 @@ public void existingFilesAreOverwritten() throws Exception { Files.write(target.toPath(), "old".getBytes()); FileOutputWriter.write( Collections.singletonList(new FileOutput(b64("new"), "REDACTED_TEXT", FileDataFormat.TXT)), - DataSourceType.BASE64, "notes.txt", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "notes.txt", folder.getRoot().getPath()); Assert.assertEquals("new", new String(Files.readAllBytes(target.toPath()))); } @@ -254,7 +254,7 @@ public void aLaterFailureLeavesEarlierFilesOnDisk() throws Exception { FileOutputWriter.write(Arrays.asList( new FileOutput(b64("ok"), "REDACTED_FILE", FileDataFormat.PDF), new FileOutput("not base64!", "ENTITIES", null)), - DataSourceType.BASE64, "invoice.pdf", folder.getRoot().getPath()); + TestDeidentifySource.BASE64, "invoice.pdf", folder.getRoot().getPath()); Assert.fail("expected SkyflowException"); } catch (SkyflowException e) { Assert.assertTrue(e.getMessage(), e.getMessage().contains("processed-entities-invoice.json")); diff --git a/common/src/test/java/com/skyflow/detect/internal/ReidentifyFileMapperTests.java b/common/src/test/java/com/skyflow/detect/internal/ReidentifyFileMapperTests.java index ec711c12..5d34c119 100644 --- a/common/src/test/java/com/skyflow/detect/internal/ReidentifyFileMapperTests.java +++ b/common/src/test/java/com/skyflow/detect/internal/ReidentifyFileMapperTests.java @@ -1,11 +1,11 @@ package com.skyflow.detect.internal; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.TestReidentifySource; import com.skyflow.detect.DetectRunStatus; import com.skyflow.detect.EntityType; import com.skyflow.detect.FileDataFormat; import com.skyflow.detect.FileOutput; -import com.skyflow.detect.RedactionLevel; +import com.skyflow.detect.TestRedactionLevel; import com.skyflow.enums.RedactionType; import com.skyflow.detect.ReidentifyFileRequest; import com.skyflow.detect.ReidentifyFileResponse; @@ -26,7 +26,7 @@ private static String wire(ReidentifyFileRequestV2 request) throws Exception { } private static ReidentifyFileResponse fromJson(String json) throws Exception { - return ReidentifyFileMapper.toResponse(ObjectMappers.JSON_MAPPER.readValue(json, ReidentifyFileResponseV2.class)); + return ReidentifyFileMapper.toResponse(ObjectMappers.JSON_MAPPER.readValue(json, ReidentifyFileResponseV2.class), TestReidentifySource::fromWire); } // ─── request ────────────────────────────────────────────────────────────── @@ -34,7 +34,7 @@ private static ReidentifyFileResponse fromJson(String json) throws Exception { @Test public void mapsRequiredFieldsAndInjectsVaultId() throws Exception { ReidentifyFileRequestV2 api = ReidentifyFileMapper.toApiRequest(ReidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64).value("W05BTUVfMV0=").build(), "vault-1"); + .dataSource(TestReidentifySource.BASE64).value("W05BTUVfMV0=").build(), "vault-1"); Assert.assertEquals("BASE64", api.getDataSource().toString()); Assert.assertEquals("W05BTUVfMV0=", api.getValue()); @@ -52,7 +52,7 @@ public void mapsRequiredFieldsAndInjectsVaultId() throws Exception { @Test public void mapsEveryDataSource() throws Exception { - for (DataSourceType source : DataSourceType.values()) { + for (TestReidentifySource source : TestReidentifySource.values()) { ReidentifyFileRequestV2 api = ReidentifyFileMapper.toApiRequest(ReidentifyFileRequest.builder() .dataSource(source).value("x").build(), "v"); Assert.assertEquals(source.name(), api.getDataSource().toString()); @@ -60,12 +60,18 @@ public void mapsEveryDataSource() throws Exception { } } + @Test + public void outputTypeTheSdkDoesNotOfferMapsToNull() throws Exception { + // the API lists SKYFLOW_ID for re-identification but the SDK enum does not offer it + Assert.assertNull(fromJson("{\"outputType\":\"SKYFLOW_ID\"}").getOutputType()); + } + @Test public void dataFormatIsSentAsLowerCaseWireValue() throws Exception { for (FileDataFormat format : new FileDataFormat[]{FileDataFormat.TXT, FileDataFormat.CSV, FileDataFormat.JSON, FileDataFormat.JSONL, FileDataFormat.XML, FileDataFormat.DOCX, FileDataFormat.XLSX, FileDataFormat.PPTX}) { ReidentifyFileRequestV2 api = ReidentifyFileMapper.toApiRequest(ReidentifyFileRequest.builder() - .dataSource(DataSourceType.SKYFLOW_ID).value("rec").dataFormat(format).build(), "v"); + .dataSource(TestReidentifySource.BASE64).value("eA==").dataFormat(format).build(), "v"); Assert.assertEquals(format.name().toLowerCase(), api.getDataFormat().get().toString()); Assert.assertTrue(wire(api).contains("\"dataFormat\":\"" + format.name().toLowerCase() + "\"")); } @@ -74,10 +80,10 @@ public void dataFormatIsSentAsLowerCaseWireValue() throws Exception { @Test public void mapsFlowvaultAndSkyvaultStyleRedactionLevels() throws Exception { ReidentifyFileRequestV2 api = ReidentifyFileMapper.toApiRequest(ReidentifyFileRequest.builder() - .dataSource(DataSourceType.PRESIGNED_URL).value("https://s3/presigned") + .dataSource(TestReidentifySource.PRESIGNED_URL).value("https://s3/presigned") .redactionLevel(Arrays.asList( - RedactionLevel.builder().tokenGroupName("names").redactionPattern("mask-middle").build(), - RedactionLevel.builder().entityName(EntityType.EMAIL_ADDRESS).redactionType(RedactionType.MASKED).build())) + TestRedactionLevel.builder().tokenGroupName("names").redactionPattern("mask-middle").build(), + TestRedactionLevel.builder().entityName(EntityType.EMAIL_ADDRESS).redactionType(RedactionType.MASKED).build())) .build(), "v"); Assert.assertEquals(2, api.getRedactionLevel().get().size()); @@ -93,7 +99,7 @@ public void mapsFlowvaultAndSkyvaultStyleRedactionLevels() throws Exception { @Test public void emptyRedactionLevelListIsSentAsEmptyArray() throws Exception { ReidentifyFileRequestV2 api = ReidentifyFileMapper.toApiRequest(ReidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64).value("x").redactionLevel(Collections.emptyList()).build(), "v"); + .dataSource(TestReidentifySource.BASE64).value("x").redactionLevel(Collections.emptyList()).build(), "v"); Assert.assertTrue(api.getRedactionLevel().get().isEmpty()); Assert.assertTrue(wire(api).contains("\"redactionLevel\":[]")); } @@ -108,7 +114,7 @@ public void mapsFullResponse() throws Exception { + "\"metrics\":{\"size\":12.5,\"wordCount\":120,\"characterCount\":800,\"pages\":2,\"slides\":0,\"duration\":1.25}}"); Assert.assertEquals(DetectRunStatus.SUCCESS, response.getStatus()); - Assert.assertEquals(DataSourceType.PRESIGNED_URL, response.getOutputType()); + Assert.assertEquals(TestReidentifySource.PRESIGNED_URL, response.getOutputType()); Assert.assertEquals(2, response.getOutput().size()); FileOutput first = response.getOutput().get(0); Assert.assertEquals("https://s3/out-1", first.getProcessedFile()); @@ -129,7 +135,7 @@ public void mapsEveryKnownStatusAndOutputType() throws Exception { for (DetectRunStatus status : DetectRunStatus.values()) { Assert.assertEquals(status, fromJson("{\"status\":\"" + status.name() + "\"}").getStatus()); } - for (DataSourceType type : DataSourceType.values()) { + for (TestReidentifySource type : TestReidentifySource.values()) { Assert.assertEquals(type, fromJson("{\"outputType\":\"" + type.name() + "\"}").getOutputType()); } } diff --git a/common/src/test/java/com/skyflow/detect/internal/ReidentifyFileValidationsTests.java b/common/src/test/java/com/skyflow/detect/internal/ReidentifyFileValidationsTests.java index 8b36827b..bcd06dd8 100644 --- a/common/src/test/java/com/skyflow/detect/internal/ReidentifyFileValidationsTests.java +++ b/common/src/test/java/com/skyflow/detect/internal/ReidentifyFileValidationsTests.java @@ -1,8 +1,9 @@ package com.skyflow.detect.internal; -import com.skyflow.detect.DataSourceType; + +import com.skyflow.detect.TestReidentifySource; import com.skyflow.detect.EntityType; -import com.skyflow.detect.RedactionLevel; +import com.skyflow.detect.TestRedactionLevel; import com.skyflow.enums.RedactionType; import com.skyflow.detect.ReidentifyFileRequest; import com.skyflow.errors.ErrorCode; @@ -17,7 +18,7 @@ public class ReidentifyFileValidationsTests { private static ReidentifyFileRequest.ReidentifyFileRequestBuilder valid() { - return ReidentifyFileRequest.builder().dataSource(DataSourceType.BASE64).value("Zm9v"); + return ReidentifyFileRequest.builder().dataSource(TestReidentifySource.BASE64).value("Zm9v"); } private static void assertRejected(ReidentifyFileRequest request, BaseErrorMessage expected) { @@ -42,7 +43,7 @@ public void rejectsMissingDataSource() { @Test public void rejectsNullOrBlankValue() { - assertRejected(ReidentifyFileRequest.builder().dataSource(DataSourceType.BASE64).build(), BaseErrorMessage.InvalidValueInReidentifyFile); + assertRejected(ReidentifyFileRequest.builder().dataSource(TestReidentifySource.BASE64).build(), BaseErrorMessage.InvalidValueInReidentifyFile); assertRejected(valid().value("").build(), BaseErrorMessage.InvalidValueInReidentifyFile); assertRejected(valid().value(" \t").build(), BaseErrorMessage.InvalidValueInReidentifyFile); } @@ -57,29 +58,29 @@ public void checksFieldsInOrderDataSourceBeforeValue() { public void acceptsMinimalAndFullValidRequests() throws SkyflowException { ReidentifyFileValidations.validateReidentifyFileRequest(valid().build()); ReidentifyFileValidations.validateReidentifyFileRequest(valid() - .dataSource(DataSourceType.PRESIGNED_URL).value("https://example.com/f") + .dataSource(TestReidentifySource.PRESIGNED_URL).value("https://example.com/f") .redactionLevel(Arrays.asList( - RedactionLevel.builder().tokenGroupName("g").redactionPattern("p").build(), - RedactionLevel.builder().entityName(EntityType.NAME).redactionType(RedactionType.REDACTED).build(), - RedactionLevel.builder().tokenGroupName("g2").redactionType(RedactionType.PLAIN_TEXT).build(), - RedactionLevel.builder().entityName(EntityType.SSN).redactionPattern("p2").build())) + TestRedactionLevel.builder().tokenGroupName("g").redactionPattern("p").build(), + TestRedactionLevel.builder().entityName(EntityType.NAME).redactionType(RedactionType.REDACTED).build(), + TestRedactionLevel.builder().tokenGroupName("g2").redactionType(RedactionType.PLAIN_TEXT).build(), + TestRedactionLevel.builder().entityName(EntityType.SSN).redactionPattern("p2").build())) .build()); } @Test public void nullOrEmptyRedactionLevelListIsAllowed() throws SkyflowException { ReidentifyFileValidations.validateReidentifyFileRequest(valid().redactionLevel(null).build()); - ReidentifyFileValidations.validateReidentifyFileRequest(valid().redactionLevel(Collections.emptyList()).build()); + ReidentifyFileValidations.validateReidentifyFileRequest(valid().redactionLevel(Collections.emptyList()).build()); } @Test public void anyRedactionLevelShapePassesThrough() throws SkyflowException { - // Every RedactionLevel field is optional on the API; the server applies its own rules. + // Every TestRedactionLevel field is optional on the API; the server applies its own rules. ReidentifyFileValidations.validateReidentifyFileRequest(valid().redactionLevel(Arrays.asList( null, - RedactionLevel.builder().build(), - RedactionLevel.builder().tokenGroupName("g").build(), - RedactionLevel.builder().tokenGroupName("g").entityName(EntityType.NAME) + TestRedactionLevel.builder().build(), + TestRedactionLevel.builder().tokenGroupName("g").build(), + TestRedactionLevel.builder().tokenGroupName("g").entityName(EntityType.NAME) .redactionPattern("p").redactionType(RedactionType.MASKED).build())).build()); } } diff --git a/common/src/test/java/com/skyflow/detect/polling/RunPollerTests.java b/common/src/test/java/com/skyflow/detect/polling/RunPollerTests.java index 1030eec6..946affeb 100644 --- a/common/src/test/java/com/skyflow/detect/polling/RunPollerTests.java +++ b/common/src/test/java/com/skyflow/detect/polling/RunPollerTests.java @@ -1,6 +1,6 @@ package com.skyflow.detect.polling; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.TestDeidentifySource; import com.skyflow.detect.DetectRunStatus; import com.skyflow.detect.GetRunResponse; import com.skyflow.detect.PollOptions; @@ -17,7 +17,7 @@ public class RunPollerTests { private static GetRunResponse run(DetectRunStatus status) { - return new GetRunResponse("run-1", status, status == DetectRunStatus.SUCCESS ? DataSourceType.BASE64 : null, + return new GetRunResponse("run-1", status, status == DetectRunStatus.SUCCESS ? TestDeidentifySource.BASE64 : null, Collections.emptyList(), null, status == DetectRunStatus.FAILED ? "boom" : null); } diff --git a/common/src/test/java/com/skyflow/serviceaccount/util/TokenTests.java b/common/src/test/java/com/skyflow/serviceaccount/util/TokenTests.java index 00fc26b4..646f8f5c 100644 --- a/common/src/test/java/com/skyflow/serviceaccount/util/TokenTests.java +++ b/common/src/test/java/com/skyflow/serviceaccount/util/TokenTests.java @@ -21,6 +21,48 @@ public static void setup() { LogUtil.setupLogger(LogLevel.DEBUG); } + private static String unsignedJwt(String payloadJson) { + Base64.Encoder enc = Base64.getUrlEncoder().withoutPadding(); + String header = enc.encodeToString("{\"alg\":\"none\"}".getBytes(StandardCharsets.UTF_8)); + String payload = enc.encodeToString(payloadJson.getBytes(StandardCharsets.UTF_8)); + return header + "." + payload + ".sig"; + } + + private static String jwtExpiringIn(long seconds) { + return unsignedJwt("{\"exp\":" + (System.currentTimeMillis() / 1000 + seconds) + "}"); + } + + @Test + public void testIsExpiringWithin_expiredTokenIsDue() { + Assert.assertTrue(Token.isExpiringWithin(jwtExpiringIn(-5), 60)); + } + + @Test + public void testIsExpiringWithin_tokenInsideWindowIsDue() { + Assert.assertTrue(Token.isExpiringWithin(jwtExpiringIn(30), 60)); + } + + @Test + public void testIsExpiringWithin_tokenBeyondWindowIsNotDue() { + Assert.assertFalse(Token.isExpiringWithin(jwtExpiringIn(600), 60)); + } + + @Test + public void testIsExpiringWithin_zeroWindowMatchesExpiryOnly() { + Assert.assertTrue(Token.isExpiringWithin(jwtExpiringIn(-1), 0)); + Assert.assertFalse(Token.isExpiringWithin(jwtExpiringIn(5), 0)); + } + + @Test + public void testIsExpiringWithin_nullEmptyMalformedAndNoExpAreNotDue() { + Assert.assertFalse(Token.isExpiringWithin(null, 60)); + Assert.assertFalse(Token.isExpiringWithin("", 60)); + Assert.assertFalse(Token.isExpiringWithin("not-a-jwt", 60)); + Assert.assertFalse(Token.isExpiringWithin("a.b", 60)); + Assert.assertFalse(Token.isExpiringWithin(unsignedJwt("{\"sub\":\"x\"}"), 60)); + Assert.assertFalse(Token.isExpiringWithin(unsignedJwt("{\"exp\":\"soon\"}"), 60)); + } + @Test public void testNoTokenForIsExpiredToken() { try { diff --git a/common/src/test/java/com/skyflow/vault/controller/BaseDetectControllerTests.java b/common/src/test/java/com/skyflow/vault/controller/BaseDetectControllerTests.java index 4b267bb3..ef4038ed 100644 --- a/common/src/test/java/com/skyflow/vault/controller/BaseDetectControllerTests.java +++ b/common/src/test/java/com/skyflow/vault/controller/BaseDetectControllerTests.java @@ -1,5 +1,8 @@ package com.skyflow.vault.controller; +import com.skyflow.detect.TestReidentifySource; +import com.skyflow.detect.ReidentifyFileSource; +import com.skyflow.detect.DeidentifyFileSource; import com.skyflow.config.BaseVaultConfig; import com.skyflow.detect.DeidentifyStringRequest; import com.skyflow.detect.DeidentifyStringResponse; @@ -8,14 +11,14 @@ import com.skyflow.detect.Entity; import com.skyflow.detect.CheckGuardrailsRequest; import com.skyflow.detect.CheckGuardrailsResponse; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.TestDeidentifySource; import com.skyflow.detect.DetectRunStatus; import com.skyflow.detect.EntityType; import com.skyflow.detect.FileDataFormat; import com.skyflow.detect.GetRunRequest; import com.skyflow.detect.GetRunResponse; import com.skyflow.detect.GuardrailsValidation; -import com.skyflow.detect.RedactionLevel; +import com.skyflow.detect.TestRedactionLevel; import com.skyflow.detect.ReidentifyStringRequest; import com.skyflow.detect.ReidentifyStringResponse; import com.skyflow.enums.Env; @@ -57,6 +60,16 @@ protected String getSdkMetrics() { return "{\"sdk_name_version\":\"test\"}"; } + @Override + protected DeidentifyFileSource toDeidentifySource(String wireValue) { + return TestDeidentifySource.fromWire(wireValue); + } + + @Override + protected ReidentifyFileSource toReidentifySource(String wireValue) { + return TestReidentifySource.fromWire(wireValue); + } + @Override protected String resolveToken() throws SkyflowException { if (tokenFailure != null) { @@ -130,7 +143,7 @@ public void reidentifyStringSendsVaultIdInBodyAndRefreshesTokenPerCall() throws ReidentifyStringResponse second = controller.reidentifyString(ReidentifyStringRequest.builder() .text("My name is [NAME_1].") .redactionLevel(Collections.singletonList( - RedactionLevel.builder().tokenGroupName("group").redactionPattern("mask").build())) + TestRedactionLevel.builder().tokenGroupName("group").redactionPattern("mask").build())) .build()); Assert.assertEquals("My name is John Doe.", first.getProcessedText()); @@ -162,7 +175,7 @@ public void getRunQueriesRunByIdScopedToVault() throws Exception { Assert.assertEquals("run-42", response.getRunId()); Assert.assertEquals(DetectRunStatus.SUCCESS, response.getStatus()); - Assert.assertEquals(DataSourceType.BASE64, response.getOutputType()); + Assert.assertEquals(TestDeidentifySource.BASE64, response.getOutputType()); Assert.assertEquals(2, response.getOutput().size()); Assert.assertEquals("QUJD", response.getOutput().get(0).getProcessedFile()); Assert.assertEquals("REDACTED_FILE", response.getOutput().get(0).getProcessedFileType()); diff --git a/docs/api_reference.md b/docs/api_reference.md index 426a1f3d..10d9ea49 100644 --- a/docs/api_reference.md +++ b/docs/api_reference.md @@ -242,7 +242,7 @@ Parameters listed with their defaults as defined in the builders. | Builder method | Default | Description | |----------------|---------|-------------| -| `dataSource(DataSourceType)` | `null` | How `value` supplies the file. See [`DataSourceType`](#datasourcetype). | +| `dataSource(DeidentifySourceType)` | `null` | How `value` supplies the file, using this SDK's enum: skyvault `BASE64`, `SKYFLOW_ID` or `PRESIGNED_URL`; flowvault `BASE64` or `PRESIGNED_URL`. Required with `value`; a `file` or `filePath` is always sent as `BASE64`. See [Detect enums](#detect-enums). | | `value(String)` | `null` | Base64 content, a skyflow id, or a presigned URL, per `dataSource`. | | `file(File)` | `null` | Local file; read and base64-encoded when sent, `dataSource` set to `BASE64`. | | `filePath(String)` | `null` | Same as `file(File)`, from a path. | @@ -267,7 +267,7 @@ Parameters listed with their defaults as defined in the builders. | Builder method | Default | Description | |----------------|---------|-------------| -| `dataSource(DataSourceType)` | _(required)_ | `BASE64`, `SKYFLOW_ID` or `PRESIGNED_URL`. | +| `dataSource(ReidentifySourceType)` | _(required)_ | `BASE64` or `PRESIGNED_URL`. Re-identification does not accept a skyflow id as the source. | | `value(String)` | _(required)_ | Base64 content, a skyflow id, or a presigned URL, per `dataSource`. | | `dataFormat(FileDataFormat)` | `null` | File format. Limited to the formats that support `VAULT_TOKEN`: `TXT`, `CSV`, `JSON`, `JSONL`, `XML`, `DOCX`, `XLSX` or `PPTX`. | | `redactionLevel(List)` | `null` | Redaction rules. See [`RedactionLevel`](#redactionlevel). | @@ -421,7 +421,7 @@ Each element returned by `getDetokenizedFields()` and `getErrors()`. |--------|-------------|-------------| | `getRunId()` | `String` | Run id, always present. | | `getStatus()` | `DetectRunStatus` | Run status when polled, `null` when submit only. See [`DetectRunStatus`](#detectrunstatus). | -| `getOutputType()` | `DataSourceType` | `BASE64` or `PRESIGNED_URL`, `null` until the run completes. | +| `getOutputType()` | `DeidentifySourceType` | `BASE64` or `PRESIGNED_URL`, `null` until the run completes. | | `getOutput()` | `List` | Every output of the run, empty until complete, never `null`. See [`FileOutput`](#fileoutput). | | `getMetrics()` | `Metrics` | Size, counts, pages, slides, duration. `null` until complete. | | `getMessage()` | `String` | Failure detail when the status is `FAILED`. | @@ -433,7 +433,7 @@ Each element returned by `getDetokenizedFields()` and `getErrors()`. | Getter | Return type | Description | |--------|-------------|-------------| | `getStatus()` | `DetectRunStatus` | Status of the re-identification. | -| `getOutputType()` | `DataSourceType` | `BASE64` or `PRESIGNED_URL`. | +| `getOutputType()` | `ReidentifySourceType` | `BASE64` or `PRESIGNED_URL`. | | `getOutput()` | `List` | Re-identified file outputs, never `null`. | | `getMetrics()` | `Metrics` | Size and counts. | @@ -577,14 +577,14 @@ Each element returned by `getDetokenizedFields()` and `getErrors()`. ### `RedactionLevel` -`com.skyflow.detect` — element of `redactionLevel()` on `ReidentifyStringRequest` and `ReidentifyFileRequest`. Each rule names one source and one replacement: `entityName` plus `redactionType` on skyvault vaults, `tokenGroupName` plus `redactionPattern` on flowvault vaults. Every field is optional on the API; the server rejects invalid combinations. +`com.skyflow.detect` — element of `redactionLevel()` on `ReidentifyStringRequest` and `ReidentifyFileRequest`. The class is defined per SDK and carries only the pair of fields its vault type uses, so the other vault's fields cannot be set. Both fields are optional on the API and are sent as given. -| Builder method | Description | -|----------------|-------------| -| `entityName(EntityType)` | Skyvault vaults: entity type the rule applies to. | -| `redactionType(RedactionType)` | Skyvault vaults: `PLAIN_TEXT`, `MASKED`, `REDACTED` or `DEFAULT`, the same `com.skyflow.enums.RedactionType` used by `get` and `detokenize`. See [`RedactionType`](#redactiontype). | -| `tokenGroupName(String)` | Flowvault vaults: token group the rule applies to. | -| `redactionPattern(String)` | Flowvault vaults: name of a redaction pattern configured in the vault. | +| SDK | Builder method | Description | +|-----|----------------|-------------| +| skyvault | `entityName(EntityType)` | Entity type the rule applies to. | +| skyvault | `redactionType(RedactionType)` | `PLAIN_TEXT`, `MASKED`, `REDACTED` or `DEFAULT`, the same `com.skyflow.enums.RedactionType` used by `get` and `detokenize`. See [`RedactionType`](#redactiontype). | +| flowvault | `tokenGroupName(String)` | Token group the rule applies to. | +| flowvault | `redactionPattern(String)` | Name of a redaction pattern configured in the vault. | ### `DetectedEntity` @@ -694,7 +694,8 @@ All in `com.skyflow.detect`. |------|--------| | `EntityType` | the 70 entity types (`NAME`, `EMAIL_ADDRESS`, `SSN`, `CREDIT_CARD`, `DOB`, ...), plus `ALL` and `NONE` | | `DeidentificationType` | `ENTITY_UNIQUE_COUNTER` _(default)_, `ENTITY_ONLY`, `VAULT_TOKEN`, `MASK`, `UNKNOWN` | -| `DataSourceType` | `BASE64`, `SKYFLOW_ID`, `PRESIGNED_URL` | +| `DeidentifySourceType` | Defined per SDK: skyvault `BASE64`, `SKYFLOW_ID`, `PRESIGNED_URL`; flowvault `BASE64`, `PRESIGNED_URL`. The `dataSource` of `deidentifyFile` and the `getOutputType()` of `DeidentifyFileResponse` and `GetRunResponse`. | +| `ReidentifySourceType` | `BASE64`, `PRESIGNED_URL` in both SDKs. The `dataSource` of `reidentifyFile` and the `getOutputType()` of `ReidentifyFileResponse`. | | `FileDataFormat` | `BMP`, `CSV`, `DCM`, `DOC`, `DOCX`, `GIF`, `JPEG`, `JPG`, `JSON`, `JSONL`, `MP3`, `PDF`, `PNG`, `PPT`, `PPTX`, `TIF`, `TIFF`, `TXT`, `WAV`, `XLS`, `XLSX`, `XML`, `ZIP` | | `DetectRunStatus` | `QUEUED`, `IN_PROGRESS`, `SUCCESS`, `FAILED`, `UNKNOWN`. `SUCCESS`, `FAILED` and `UNKNOWN` end polling. | | `ReturnEntitiesType` | `EXCLUDE_SENSITIVE_DATA` _(default)_, `ALL`, `NONE` | diff --git a/flowvault/README.md b/flowvault/README.md index 48f37775..d1727eb3 100644 --- a/flowvault/README.md +++ b/flowvault/README.md @@ -1762,7 +1762,7 @@ File de-identification is asynchronous on the API: the submit call returns a run - **Submit only** (default): one call, returns `getRunId()`. `getStatus()` is `null` and nothing is written to disk. Retrieve the result later with [`getRun`](#get-run). - **Submit and poll**: set `pollOptions`. The SDK polls the run with exponential backoff (1, 2, 4, 8, 16, then 32 seconds between lookups) until it is `SUCCESS`, `FAILED` or `UNKNOWN`, or until `waitTime` (default 60 s, max 300) or `maxAttempts` (default 23, max 23) is used up. If the budget runs out the response carries the run id with `IN_PROGRESS` or `QUEUED` so you can resume with `getRun`. -The file is supplied in one of three ways: `dataSource` plus `value` (`BASE64` content, a `SKYFLOW_ID`, or a `PRESIGNED_URL`), or `file(File)` / `filePath(String)`, which read and encode a local file and infer `dataFormat` from its extension. Supply exactly one of `value`, `file` or `filePath`. `configurationId` and `configuration` follow the same rules as for strings: both optional, sent as supplied, vault defaults when neither is set, and both together rejected by the API (see [Before you start](#before-you-start)). +The file is supplied in exactly one of three ways: `filePath(String)`, `file(File)`, or `value` plus a `DeidentifySourceType` `dataSource` (`BASE64` or `PRESIGNED_URL`; flowvault vaults do not accept a skyflow id as the source). `file` and `filePath` read a local file when the request is sent, send it as `BASE64` and infer `dataFormat` from its extension. Passing more than one of `filePath`, `file` and `value`, or none of them, is rejected before any network call. `configurationId` and `configuration` follow the same rules as for strings: both optional, sent as supplied, vault defaults when neither is set, and both together rejected by the API (see [Before you start](#before-you-start)). `dataFormat` accepts every `FileDataFormat`: `BMP`, `CSV`, `DCM`, `DOC`, `DOCX`, `GIF`, `JPEG`, `JPG`, `JSON`, `JSONL`, `MP3`, `PDF`, `PNG`, `PPT`, `PPTX`, `TIF`, `TIFF`, `TXT`, `WAV`, `XLS`, `XLSX`, `XML` and `ZIP`. The `VAULT_TOKEN` de-identification type is supported only for `TXT`, `CSV`, `JSON`, `JSONL`, `XML`, `DOCX`, `XLSX` and `PPTX`. @@ -1801,7 +1801,7 @@ switch (response.getStatus()) { } ``` -Outputs are written only when polling reached `SUCCESS`, the request was built from a local file, and the run delivered them as base64. A request built from a `SKYFLOW_ID` or `PRESIGNED_URL` has no local name to derive from, and presigned-URL outputs are not fetched, so in those cases read `getOutput()` and write the files yourself. Files are named `processed-.`, with the entities output as `processed-entities-.json` (and `processed-object-entities-.json` for object entities); `outputDirectory` is created if missing and defaults to the working directory. Polling blocks the calling thread. +Outputs are written only when polling reached `SUCCESS`, the request was built from a local file, and the run delivered them as base64. A request built from a `SKYFLOW_ID` or `PRESIGNED_URL` value has no local name to derive from, and presigned-URL outputs are not fetched, so in those cases read `getOutput()` and write the files yourself. Files are named `processed-.`, with the entities output as `processed-entities-.json` (and `processed-object-entities-.json` for object entities); `outputDirectory` is created if missing and defaults to the working directory. Polling blocks the calling thread. ### Polling from a worker thread @@ -1887,18 +1887,18 @@ System.out.println(response.getProcessedText()); // My name is J**n D*e. ``` -A `RedactionLevel` names one source and one replacement. On flowvault vaults the source is a `tokenGroupName` and the replacement a `redactionPattern` configured in the vault. The `entityName` and `redactionType` fields are the skyvault equivalents and do not apply to flowvault vaults. Every field is optional on the API, so the SDK passes the list through as given and the server rejects invalid combinations. +A `RedactionLevel` names one source and one replacement. On flowvault vaults the source is a `tokenGroupName` and the replacement a `redactionPattern` configured in the vault. The flowvault SDK's `RedactionLevel` has only these two fields; the skyvault SDK's has `entityName` and `redactionType` instead. Both are optional on the API and are sent as given. ## Reidentify file -`reidentifyFile` is synchronous: the re-identified file comes back in the response, not through a run id. `ReidentifyFileRequest` takes a `dataSource` (`BASE64`, `SKYFLOW_ID` or `PRESIGNED_URL`), the matching `value`, an optional `dataFormat` and the same optional `RedactionLevel` rules as `reidentifyString`. `dataFormat` is limited to the formats that support `VAULT_TOKEN`: `TXT`, `CSV`, `JSON`, `JSONL`, `XML`, `DOCX`, `XLSX` and `PPTX`. +`reidentifyFile` is synchronous: the re-identified file comes back in the response, not through a run id. `ReidentifyFileRequest` takes a `dataSource` (`ReidentifySourceType.BASE64` or `PRESIGNED_URL`; re-identification does not accept a skyflow id as the source), the matching `value`, an optional `dataFormat` and the same optional `RedactionLevel` rules as `reidentifyString`. `dataFormat` is limited to the formats that support `VAULT_TOKEN`: `TXT`, `CSV`, `JSON`, `JSONL`, `XML`, `DOCX`, `XLSX` and `PPTX`. ```java byte[] content = Files.readAllBytes(Paths.get("tokenized.txt")); ReidentifyFileResponse response = skyflowClient.detect().reidentifyFile( ReidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64) + .dataSource(ReidentifySourceType.BASE64) .value(Base64.getEncoder().encodeToString(content)) .dataFormat(FileDataFormat.TXT) .redactionLevel(Collections.singletonList( diff --git a/flowvault/api-report/skyflow-flowvault-java.baseline.jar b/flowvault/api-report/skyflow-flowvault-java.baseline.jar index a131a5ee..7b870015 100644 Binary files a/flowvault/api-report/skyflow-flowvault-java.baseline.jar and b/flowvault/api-report/skyflow-flowvault-java.baseline.jar differ diff --git a/flowvault/pom.xml b/flowvault/pom.xml index 75b0d4d7..a110822a 100644 --- a/flowvault/pom.xml +++ b/flowvault/pom.xml @@ -12,7 +12,7 @@ skyflow-flowvault-java - 1.2.0-dev.7bfc279c + 1.3.0-dev.1b7228dd jar ${project.groupId}:${project.artifactId} Skyflow FlowVault SDK for the Java programming language diff --git a/flowvault/samples/src/main/java/com/example/detect/GetRunSample.java b/flowvault/samples/src/main/java/com/example/detect/GetRunSample.java index 9d704f25..1d521d71 100644 --- a/flowvault/samples/src/main/java/com/example/detect/GetRunSample.java +++ b/flowvault/samples/src/main/java/com/example/detect/GetRunSample.java @@ -3,7 +3,7 @@ import com.skyflow.Skyflow; import com.skyflow.config.Credentials; import com.skyflow.config.VaultConfig; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.DeidentifySourceType; import com.skyflow.detect.DetectRunStatus; import com.skyflow.detect.FileOutput; import com.skyflow.detect.GetRunRequest; @@ -46,7 +46,7 @@ public static void main(String[] args) throws Exception { System.out.println("Run " + run.getRunId() + " status: " + run.getStatus()); if (run.getStatus() == DetectRunStatus.SUCCESS) { for (FileOutput output : run.getOutput()) { - if (run.getOutputType() != DataSourceType.BASE64) { + if (run.getOutputType() != DeidentifySourceType.BASE64) { // PRESIGNED_URL outputs carry a URL, not base64: download them instead of decoding. System.out.println(output.getProcessedFileType() + " available at " + output.getProcessedFile()); continue; diff --git a/flowvault/samples/src/main/java/com/example/detect/ReidentifyFileSample.java b/flowvault/samples/src/main/java/com/example/detect/ReidentifyFileSample.java index 3e0cd0d8..020fbf52 100644 --- a/flowvault/samples/src/main/java/com/example/detect/ReidentifyFileSample.java +++ b/flowvault/samples/src/main/java/com/example/detect/ReidentifyFileSample.java @@ -3,7 +3,7 @@ import com.skyflow.Skyflow; import com.skyflow.config.Credentials; import com.skyflow.config.VaultConfig; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.ReidentifySourceType; import com.skyflow.detect.DetectRunStatus; import com.skyflow.detect.FileDataFormat; import com.skyflow.detect.FileOutput; @@ -45,7 +45,7 @@ public static void main(String[] args) throws Exception { try { ReidentifyFileResponse response = skyflowClient.detect().reidentifyFile( ReidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64) + .dataSource(ReidentifySourceType.BASE64) .value(base64) .dataFormat(FileDataFormat.TXT) // Optional: render one token group with a named redaction pattern instead of its plain value. @@ -61,7 +61,7 @@ public static void main(String[] args) throws Exception { int index = 0; for (FileOutput output : response.getOutput()) { index++; - if (response.getOutputType() != DataSourceType.BASE64) { + if (response.getOutputType() != ReidentifySourceType.BASE64) { // PRESIGNED_URL outputs carry a URL, not base64: download them instead of decoding. System.out.println(output.getProcessedFileType() + " available at " + output.getProcessedFile()); continue; diff --git a/flowvault/src/main/java/com/skyflow/VaultClient.java b/flowvault/src/main/java/com/skyflow/VaultClient.java index 8a7264c9..e1d90268 100644 --- a/flowvault/src/main/java/com/skyflow/VaultClient.java +++ b/flowvault/src/main/java/com/skyflow/VaultClient.java @@ -13,7 +13,6 @@ import okhttp3.ConnectionPool; import okhttp3.OkHttpClient; -import okhttp3.Request; public class VaultClient extends BaseVaultClient { private final ApiClientBuilder apiClientBuilder; @@ -167,12 +166,9 @@ protected void updateExecutorInHTTP() throws SkyflowException { // OUTER: retries. Must wrap the auth interceptor so each attempt re-reads the // (possibly refreshed) bearer token rather than replaying a stale one. .addInterceptor(new SkyflowRetryInterceptor(maxRetries, initialRetryDelayMillis, maxRetryDelayMillis)) - .addInterceptor(chain -> { // INNER: auth - Request requestWithAuth = chain.request().newBuilder() - .header("Authorization", "Bearer " + this.token) - .build(); - return chain.proceed(requestWithAuth); - }); + // INNER: auth, shared with skyvault. Attaches the token and, on a 401 for an + // expired SDK-generated token, regenerates once and resends once. + .addInterceptor(authInterceptor(this::getToken)); if (connectTimeout != null) { httpBuilder.connectTimeout(connectTimeout, TimeUnit.SECONDS); } diff --git a/flowvault/src/main/java/com/skyflow/detect/DeidentifySourceType.java b/flowvault/src/main/java/com/skyflow/detect/DeidentifySourceType.java new file mode 100644 index 00000000..86f5e13f --- /dev/null +++ b/flowvault/src/main/java/com/skyflow/detect/DeidentifySourceType.java @@ -0,0 +1,11 @@ +package com.skyflow.detect; + +/** + * How the input file is supplied to {@code deidentifyFile} on a flowvault vault: base64 content or an S3 presigned URL. Flowvault vaults do not accept a skyflow id as the + * source, so there is no {@code SKYFLOW_ID} here. + * Also the {@code getOutputType()} of {@code DeidentifyFileResponse} and {@code GetRunResponse}. + */ +public enum DeidentifySourceType implements DeidentifyFileSource { + BASE64, + PRESIGNED_URL; +} diff --git a/flowvault/src/main/java/com/skyflow/detect/RedactionLevel.java b/flowvault/src/main/java/com/skyflow/detect/RedactionLevel.java new file mode 100644 index 00000000..031859dc --- /dev/null +++ b/flowvault/src/main/java/com/skyflow/detect/RedactionLevel.java @@ -0,0 +1,96 @@ +package com.skyflow.detect; + +/** + * Controls how one group of tokens is rendered when re-identifying on a flowvault vault: the {@code tokenGroupName} + * the rule applies to and the named {@code redactionPattern}, configured in the vault, to render its values with. + * Element of {@code redactionLevel} on {@code ReidentifyStringRequest} and {@code ReidentifyFileRequest}. Both + * fields are optional on the API and are sent as given. + */ +public final class RedactionLevel extends RedactionRule { + private final String tokenGroupName; + private final String redactionPattern; + + private RedactionLevel(RedactionLevelBuilder builder) { + this.tokenGroupName = builder.tokenGroupName; + this.redactionPattern = builder.redactionPattern; + } + + /** + * @return a new builder + */ + public static RedactionLevelBuilder builder() { + return new RedactionLevelBuilder(); + } + + /** + * @return Token group the rule applies to. + */ + public String getTokenGroupName() { + return tokenGroupName; + } + + /** + * @return Name of the redaction pattern, configured in the vault, applied to that group's values. + */ + public String getRedactionPattern() { + return redactionPattern; + } + + @Override + String apiEntityName() { + return null; + } + + @Override + String apiRedactionType() { + return null; + } + + @Override + String apiTokenGroupName() { + return tokenGroupName; + } + + @Override + String apiRedactionPattern() { + return redactionPattern; + } + + @Override + public String toString() { + return "RedactionLevel{" + + "tokenGroupName=" + tokenGroupName + + ", redactionPattern=" + redactionPattern + + '}'; + } + + public static final class RedactionLevelBuilder { + private String tokenGroupName; + private String redactionPattern; + + private RedactionLevelBuilder() { + } + + /** + * @param tokenGroupName Token group the rule applies to. + * @return this builder + */ + public RedactionLevelBuilder tokenGroupName(String tokenGroupName) { + this.tokenGroupName = tokenGroupName; + return this; + } + + /** + * @param redactionPattern Name of a redaction pattern configured in the vault. + * @return this builder + */ + public RedactionLevelBuilder redactionPattern(String redactionPattern) { + this.redactionPattern = redactionPattern; + return this; + } + + public RedactionLevel build() { + return new RedactionLevel(this); + } + } +} diff --git a/flowvault/src/main/java/com/skyflow/detect/ReidentifySourceType.java b/flowvault/src/main/java/com/skyflow/detect/ReidentifySourceType.java new file mode 100644 index 00000000..c4e0b5bc --- /dev/null +++ b/flowvault/src/main/java/com/skyflow/detect/ReidentifySourceType.java @@ -0,0 +1,11 @@ +package com.skyflow.detect; + +/** + * How the input file is supplied to {@code reidentifyFile} on a flowvault vault: base64 content or an S3 presigned + * URL. Re-identification does not accept a skyflow id as the source. Also the {@code getOutputType()} of + * {@code ReidentifyFileResponse}. + */ +public enum ReidentifySourceType implements ReidentifyFileSource { + BASE64, + PRESIGNED_URL; +} diff --git a/flowvault/src/main/java/com/skyflow/vault/controller/DetectController.java b/flowvault/src/main/java/com/skyflow/vault/controller/DetectController.java index 573eb349..90ce2206 100644 --- a/flowvault/src/main/java/com/skyflow/vault/controller/DetectController.java +++ b/flowvault/src/main/java/com/skyflow/vault/controller/DetectController.java @@ -2,6 +2,10 @@ import com.skyflow.config.Credentials; import com.skyflow.config.VaultConfig; +import com.skyflow.detect.DeidentifyFileSource; +import com.skyflow.detect.DeidentifySourceType; +import com.skyflow.detect.ReidentifyFileSource; +import com.skyflow.detect.ReidentifySourceType; import com.skyflow.errors.SkyflowException; import com.skyflow.utils.Utils; @@ -35,6 +39,16 @@ protected String resolveVaultUrl() throws SkyflowException { return vaultUrl; } + @Override + protected DeidentifyFileSource toDeidentifySource(String wireValue) { + return enumOrNull(DeidentifySourceType.class, wireValue); + } + + @Override + protected ReidentifyFileSource toReidentifySource(String wireValue) { + return enumOrNull(ReidentifySourceType.class, wireValue); + } + @Override protected String getSdkMetrics() { return Utils.getMetrics().toString(); diff --git a/flowvault/src/test/java/com/skyflow/AuthInterceptorTests.java b/flowvault/src/test/java/com/skyflow/AuthInterceptorTests.java index 631a1120..a53227f1 100644 --- a/flowvault/src/test/java/com/skyflow/AuthInterceptorTests.java +++ b/flowvault/src/test/java/com/skyflow/AuthInterceptorTests.java @@ -15,7 +15,9 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Base64; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; /** * The auth interceptor installed by {@link VaultClient#updateExecutorInHTTP()}. @@ -104,4 +106,103 @@ public void testAuthInterceptor_returnsTheChainResponseUntouched() throws Skyflo Assert.assertEquals(503, authInterceptorOf(client).intercept(chain).code()); Assert.assertEquals("auth must not retry - that is the outer interceptor's job", 1, chain.calls()); } + + // ── refresh-and-resend on 401, wired through the flowvault client ──────── + + private static String jwtExpiringIn(long seconds) { + Base64.Encoder enc = Base64.getUrlEncoder().withoutPadding(); + String header = enc.encodeToString("{\"alg\":\"none\"}".getBytes(StandardCharsets.UTF_8)); + String payload = enc.encodeToString(("{\"exp\":" + (System.currentTimeMillis() / 1000 + seconds) + "}") + .getBytes(StandardCharsets.UTF_8)); + return header + "." + payload + ".sig"; + } + + /** Flowvault client with mintable credentials and a scripted token source. */ + private static final class MintingClient extends VaultClient { + final AtomicInteger generations = new AtomicInteger(); + final String fresh; + + MintingClient(String fresh) throws SkyflowException { + super(configWithCredentialsString(), null); + this.fresh = fresh; + this.finalCredentials = getVaultConfig().getCredentials(); + } + + @Override + protected String generateBearerToken(com.skyflow.config.BaseCredentials credentials) { + generations.incrementAndGet(); + return fresh; + } + } + + private static VaultConfig configWithCredentialsString() { + Credentials credentials = new Credentials(); + credentials.setCredentialsString("{\"clientID\":\"c\"}"); + VaultConfig config = config(); + config.setCredentials(credentials); + return config; + } + + @Test + public void testAuthInterceptor_regeneratesOnceAndResendsOn401ForExpiredToken() throws SkyflowException, IOException { + MintingClient client = new MintingClient("fresh-token"); + client.token = jwtExpiringIn(-1); + FakeChain chain = new FakeChain(401, 200); + + Assert.assertEquals(200, authInterceptorOf(client).intercept(chain).code()); + Assert.assertEquals(2, chain.calls()); + Assert.assertEquals("Bearer fresh-token", chain.lastProceeded().header("Authorization")); + Assert.assertEquals("the cache must hold the new token for the batches that follow", "fresh-token", client.token); + Assert.assertEquals(1, client.generations.get()); + Assert.assertTrue("the rejected response must be closed before the resend", chain.bodies().get(0).isClosed()); + } + + @Test + public void testAuthInterceptor_401ForTokenWithLifeLeftIsReturnedWithoutRegenerating() throws SkyflowException, IOException { + MintingClient client = new MintingClient("fresh-token"); + String live = jwtExpiringIn(900); + client.token = live; + FakeChain chain = new FakeChain(401, 200); + + Assert.assertEquals(401, authInterceptorOf(client).intercept(chain).code()); + Assert.assertEquals(1, chain.calls()); + Assert.assertEquals(0, client.generations.get()); + Assert.assertEquals(live, client.token); + } + + @Test + public void testAuthInterceptor_401WithStaticTokenIsReturnedAsToday() throws SkyflowException, IOException { + Credentials credentials = new Credentials(); + credentials.setToken(jwtExpiringIn(-1)); + VaultConfig config = config(); + config.setCredentials(credentials); + VaultClient client = new VaultClient(config, null); + client.setBearerToken(); + FakeChain chain = new FakeChain(401, 200); + + Assert.assertEquals(401, authInterceptorOf(client).intercept(chain).code()); + Assert.assertEquals(1, chain.calls()); + } + + @Test + public void testAuthInterceptor_resendThatAlsoGets401IsNotRetriedAgain() throws SkyflowException, IOException { + MintingClient client = new MintingClient("fresh-token"); + client.token = jwtExpiringIn(-1); + FakeChain chain = new FakeChain(401, 401, 200); + + Assert.assertEquals(401, authInterceptorOf(client).intercept(chain).code()); + Assert.assertEquals(2, chain.calls()); + Assert.assertEquals(1, client.generations.get()); + } + + @Test + public void testAuthInterceptor_sitsInsideTheTransportRetryInterceptor() throws SkyflowException { + VaultClient client = new VaultClient(config(), null); + client.updateExecutorInHTTP(); + List interceptors = client.sharedHttpClient.interceptors(); + + Assert.assertEquals(2, interceptors.size()); + Assert.assertTrue("transport retries must wrap auth so each attempt re-reads the token", + interceptors.get(0) instanceof SkyflowRetryInterceptor); + } } diff --git a/flowvault/src/test/java/com/skyflow/detect/DetectSourceTypesTests.java b/flowvault/src/test/java/com/skyflow/detect/DetectSourceTypesTests.java new file mode 100644 index 00000000..589b5aca --- /dev/null +++ b/flowvault/src/test/java/com/skyflow/detect/DetectSourceTypesTests.java @@ -0,0 +1,39 @@ +package com.skyflow.detect; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class DetectSourceTypesTests { + + private static List names(Enum[] values) { + List names = new ArrayList<>(); + for (Enum value : values) { + names.add(value.name()); + } + return names; + } + + @Test + public void deidentifySourcesOfferedOnFlowvault() { + Assert.assertEquals(Arrays.asList("BASE64", "PRESIGNED_URL"), names(DeidentifySourceType.values())); + DeidentifyFileSource asSource = DeidentifySourceType.BASE64; + Assert.assertEquals("BASE64", asSource.name()); + } + + @Test + public void reidentifySourcesNeverIncludeSkyflowId() { + Assert.assertEquals(Arrays.asList("BASE64", "PRESIGNED_URL"), names(ReidentifySourceType.values())); + try { + ReidentifySourceType.valueOf("SKYFLOW_ID"); + Assert.fail("re-identification does not accept a skyflow id"); + } catch (IllegalArgumentException expected) { + // not offered + } + ReidentifyFileSource asSource = ReidentifySourceType.PRESIGNED_URL; + Assert.assertEquals("PRESIGNED_URL", asSource.name()); + } +} diff --git a/flowvault/src/test/java/com/skyflow/detect/RedactionLevelTests.java b/flowvault/src/test/java/com/skyflow/detect/RedactionLevelTests.java new file mode 100644 index 00000000..68c1026f --- /dev/null +++ b/flowvault/src/test/java/com/skyflow/detect/RedactionLevelTests.java @@ -0,0 +1,27 @@ +package com.skyflow.detect; + +import org.junit.Assert; +import org.junit.Test; + +public class RedactionLevelTests { + + @Test + public void flowvaultRuleCarriesTokenGroupAndPatternOnly() { + RedactionLevel rule = RedactionLevel.builder().tokenGroupName("names").redactionPattern("mask").build(); + Assert.assertEquals("names", rule.getTokenGroupName()); + Assert.assertEquals("mask", rule.getRedactionPattern()); + + com.skyflow.generated.common.rest.types.RedactionLevel api = RedactionRules.toApi(rule); + Assert.assertEquals("names", api.getTokenGroupName().get()); + Assert.assertEquals("mask", api.getRedactionPattern().get()); + Assert.assertFalse(api.getEntityName().isPresent()); + Assert.assertFalse(api.getRedactionType().isPresent()); + } + + @Test + public void unsetFieldsAreNotSent() { + com.skyflow.generated.common.rest.types.RedactionLevel api = RedactionRules.toApi(RedactionLevel.builder().build()); + Assert.assertFalse(api.getTokenGroupName().isPresent()); + Assert.assertFalse(api.getRedactionPattern().isPresent()); + } +} diff --git a/flowvault/src/test/java/com/skyflow/utils/FakeChain.java b/flowvault/src/test/java/com/skyflow/utils/FakeChain.java index 97012f13..b803fe1c 100644 --- a/flowvault/src/test/java/com/skyflow/utils/FakeChain.java +++ b/flowvault/src/test/java/com/skyflow/utils/FakeChain.java @@ -132,5 +132,9 @@ public BufferedSource source() { public void close() { closed = true; } + + public boolean isClosed() { + return closed; + } } } diff --git a/flowvault/src/test/java/com/skyflow/vault/controller/DetectControllerTests.java b/flowvault/src/test/java/com/skyflow/vault/controller/DetectControllerTests.java index 0af983c8..e81bf9e3 100644 --- a/flowvault/src/test/java/com/skyflow/vault/controller/DetectControllerTests.java +++ b/flowvault/src/test/java/com/skyflow/vault/controller/DetectControllerTests.java @@ -1,11 +1,12 @@ package com.skyflow.vault.controller; +import com.skyflow.detect.ReidentifySourceType; import com.skyflow.Skyflow; import com.skyflow.config.Credentials; import com.skyflow.config.VaultConfig; import com.skyflow.detect.CheckGuardrailsRequest; import com.skyflow.detect.CheckGuardrailsResponse; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.DeidentifySourceType; import com.skyflow.detect.DeidentificationType; import com.skyflow.detect.DeidentifyFileRequest; import com.skyflow.detect.DeidentifyFileResponse; @@ -156,7 +157,7 @@ public void reidentifyFileValidatesBeforeResolvingCredentials() throws SkyflowEx ReidentifyFileRequest[] bad = { null, ReidentifyFileRequest.builder().value("Zm9v").build(), - ReidentifyFileRequest.builder().dataSource(DataSourceType.BASE64).value(" ").build(), + ReidentifyFileRequest.builder().dataSource(ReidentifySourceType.BASE64).value(" ").build(), }; BaseErrorMessage[] expected = { BaseErrorMessage.ReidentifyFileRequestNull, @@ -179,7 +180,7 @@ public void reidentifyFileMissingCredentialsSurfaceWithoutNetworkCall() throws S DetectController controller = new DetectController(vaultConfig("vault-a"), emptyCredentials()); try { controller.reidentifyFile(ReidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64).value("Zm9v").build()); + .dataSource(ReidentifySourceType.BASE64).value("Zm9v").build()); Assert.fail("expected SkyflowException"); } catch (SkyflowException e) { Assert.assertEquals(ErrorCode.INVALID_INPUT.getCode(), e.getHttpCode()); @@ -199,7 +200,7 @@ public void reidentifyFileSendsScopedRequestAndMapsResponse() throws Exception { + "\"metrics\":{\"size\":0.5,\"wordCount\":3,\"characterCount\":12}}")); ReidentifyFileResponse response = controllerFor(server).reidentifyFile(ReidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64) + .dataSource(ReidentifySourceType.BASE64) .value("W05BTUVfMV0=") .dataFormat(FileDataFormat.TXT) .redactionLevel(Arrays.asList( @@ -207,7 +208,7 @@ public void reidentifyFileSendsScopedRequestAndMapsResponse() throws Exception { .build()); Assert.assertEquals(DetectRunStatus.SUCCESS, response.getStatus()); - Assert.assertEquals(DataSourceType.BASE64, response.getOutputType()); + Assert.assertEquals(ReidentifySourceType.BASE64, response.getOutputType()); Assert.assertEquals(1, response.getOutput().size()); Assert.assertEquals("SGVsbG8=", response.getOutput().get(0).getProcessedFile()); Assert.assertEquals("REIDENTIFIED_FILE", response.getOutput().get(0).getProcessedFileType()); @@ -238,7 +239,7 @@ public void reidentifyFileToleratesSparseAndUnknownResponseValues() throws Excep try { server.enqueue(new MockResponse().setResponseCode(200).setBody("{\"status\":\"SOMETHING_NEW\"}")); ReidentifyFileResponse response = controllerFor(server).reidentifyFile(ReidentifyFileRequest.builder() - .dataSource(DataSourceType.SKYFLOW_ID).value("rec-1").build()); + .dataSource(ReidentifySourceType.PRESIGNED_URL).value("https://example.com/tokenized.txt").build()); Assert.assertEquals(DetectRunStatus.UNKNOWN, response.getStatus()); Assert.assertNull(response.getOutputType()); Assert.assertTrue(response.getOutput().isEmpty()); @@ -261,7 +262,7 @@ public void reidentifyFileMapsApiErrors() throws Exception { + "\"http_status\":\"Bad Request\",\"details\":[]}}")); try { controllerFor(server).reidentifyFile(ReidentifyFileRequest.builder() - .dataSource(DataSourceType.PRESIGNED_URL).value("https://example.com/f").build()); + .dataSource(ReidentifySourceType.PRESIGNED_URL).value("https://example.com/f").build()); Assert.fail("expected SkyflowException"); } catch (SkyflowException e) { Assert.assertEquals(400, e.getHttpCode()); @@ -403,7 +404,7 @@ public void getRunRoundTripQueriesByIdScopedToVault() throws Exception { GetRunResponse done = controller.getRun(GetRunRequest.builder().runId(" run-42 ").build()); Assert.assertEquals("run-42", done.getRunId()); Assert.assertEquals(DetectRunStatus.SUCCESS, done.getStatus()); - Assert.assertEquals(DataSourceType.BASE64, done.getOutputType()); + Assert.assertEquals(DeidentifySourceType.BASE64, done.getOutputType()); Assert.assertEquals("UERG", done.getOutput().get(0).getProcessedFile()); Assert.assertEquals("REDACTED_FILE", done.getOutput().get(0).getProcessedFileType()); Assert.assertEquals(FileDataFormat.PDF, done.getOutput().get(0).getProcessedFileExtension()); @@ -506,7 +507,7 @@ public void deidentifyFileSubmitOnlyReturnsRunIdWithoutPolling() throws Exceptio server.enqueue(new MockResponse().setResponseCode(200).setBody("{\"runId\":\"run-42\"}")); DeidentifyFileResponse response = controllerFor(server).deidentifyFile(DeidentifyFileRequest.builder() - .dataSource(DataSourceType.SKYFLOW_ID).value("rec-1").dataFormat(FileDataFormat.PDF) + .dataSource(DeidentifySourceType.PRESIGNED_URL).value("https://example.com/in.pdf").dataFormat(FileDataFormat.PDF) .configurationId("cfg-1").build()); Assert.assertEquals("run-42", response.getRunId()); @@ -516,8 +517,8 @@ public void deidentifyFileSubmitOnlyReturnsRunIdWithoutPolling() throws Exceptio RecordedRequest recorded = server.takeRequest(); assertCommonHeaders(recorded, "POST", "/v2/detect/deidentify/file"); String body = recorded.getBody().readUtf8(); - Assert.assertTrue(body, body.contains("\"dataSource\":\"SKYFLOW_ID\"")); - Assert.assertTrue(body, body.contains("\"value\":\"rec-1\"")); + Assert.assertTrue(body, body.contains("\"dataSource\":\"PRESIGNED_URL\"")); + Assert.assertTrue(body, body.contains("\"value\":\"https://example.com/in.pdf\"")); Assert.assertTrue(body, body.contains("\"dataFormat\":\"pdf\"")); Assert.assertTrue(body, body.contains("\"configurationId\":\"cfg-1\"")); Assert.assertEquals("only the submit call was made", 1, server.getRequestCount()); @@ -551,7 +552,7 @@ public void deidentifyFilePollsToSuccessAndWritesOutputsForLocalFile() throws Ex Assert.assertEquals("run-7", response.getRunId()); Assert.assertEquals(DetectRunStatus.SUCCESS, response.getStatus()); - Assert.assertEquals(DataSourceType.BASE64, response.getOutputType()); + Assert.assertEquals(DeidentifySourceType.BASE64, response.getOutputType()); Assert.assertEquals(2, response.getOutput().size()); Assert.assertEquals(Integer.valueOf(3), response.getMetrics().getWordCount()); Assert.assertEquals("Pay [NAME_1]", new String(Files.readAllBytes(new File(outDir, "processed-invoice.txt").toPath()), "UTF-8")); @@ -579,7 +580,7 @@ public void deidentifyFileReturnsRunIdWithStatusWhenBudgetRunsOut() throws Excep server.enqueue(new MockResponse().setResponseCode(200).setBody("{\"status\":\"IN_PROGRESS\"}")); DeidentifyFileResponse response = controllerFor(server).deidentifyFile(DeidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64).value("Zm9v") + .dataSource(DeidentifySourceType.BASE64).value("Zm9v") .pollOptions(PollOptions.builder().maxAttempts(1).build()).build()); Assert.assertEquals("run-8", response.getRunId()); @@ -600,7 +601,7 @@ public void deidentifyFileSurfacesFailedRunMessage() throws Exception { server.enqueue(new MockResponse().setResponseCode(200).setBody("{\"status\":\"FAILED\",\"message\":\"unsupported file\"}")); DeidentifyFileResponse response = controllerFor(server).deidentifyFile(DeidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64).value("Zm9v").pollOptions(PollOptions.builder().build()).build()); + .dataSource(DeidentifySourceType.BASE64).value("Zm9v").pollOptions(PollOptions.builder().build()).build()); Assert.assertEquals(DetectRunStatus.FAILED, response.getStatus()); Assert.assertEquals("unsupported file", response.getMessage()); @@ -619,7 +620,7 @@ public void deidentifyFileSubmitErrorMapsToSkyflowException() throws Exception { .setBody("{\"error\":{\"grpc_code\":3,\"http_code\":400,\"message\":\"configuration not found\",\"http_status\":\"Bad Request\",\"details\":[]}}")); try { controllerFor(server).deidentifyFile(DeidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64).value("Zm9v").configurationId("missing").build()); + .dataSource(DeidentifySourceType.BASE64).value("Zm9v").configurationId("missing").build()); Assert.fail("expected SkyflowException"); } catch (SkyflowException e) { Assert.assertEquals(400, e.getHttpCode()); diff --git a/skyvault/README.md b/skyvault/README.md index 99260df5..8ed1a551 100644 --- a/skyvault/README.md +++ b/skyvault/README.md @@ -1795,7 +1795,7 @@ File de-identification is asynchronous on the API: the submit call returns a run - **Submit only** (default): one call, returns `getRunId()`. `getStatus()` is `null` and nothing is written to disk. Retrieve the result later with [`getRun`](#get-run). - **Submit and poll**: set `pollOptions`. The SDK polls the run with exponential backoff (1, 2, 4, 8, 16, then 32 seconds between lookups) until it is `SUCCESS`, `FAILED` or `UNKNOWN`, or until `waitTime` (default 60 s, max 300) or `maxAttempts` (default 23, max 23) is used up. If the budget runs out the response carries the run id with `IN_PROGRESS` or `QUEUED` so you can resume with `getRun`. -The file is supplied in one of three ways: `dataSource` plus `value` (`BASE64` content, a `SKYFLOW_ID`, or a `PRESIGNED_URL`), or `file(File)` / `filePath(String)`, which read and encode a local file and infer `dataFormat` from its extension. Supply exactly one of `value`, `file` or `filePath`. `configurationId` and `configuration` follow the same rules as for strings: both optional, sent as supplied, vault defaults when neither is set, and both together rejected by the API (see [Before you start](#before-you-start)). +The file is supplied in exactly one of three ways: `filePath(String)`, `file(File)`, or `value` plus a `DeidentifySourceType` `dataSource` (`BASE64`, the `SKYFLOW_ID` of a file record in the vault, or `PRESIGNED_URL`). `file` and `filePath` read a local file when the request is sent, send it as `BASE64` and infer `dataFormat` from its extension. Passing more than one of `filePath`, `file` and `value`, or none of them, is rejected before any network call. `configurationId` and `configuration` follow the same rules as for strings: both optional, sent as supplied, vault defaults when neither is set, and both together rejected by the API (see [Before you start](#before-you-start)). `dataFormat` accepts every `FileDataFormat`: `BMP`, `CSV`, `DCM`, `DOC`, `DOCX`, `GIF`, `JPEG`, `JPG`, `JSON`, `JSONL`, `MP3`, `PDF`, `PNG`, `PPT`, `PPTX`, `TIF`, `TIFF`, `TXT`, `WAV`, `XLS`, `XLSX`, `XML` and `ZIP`. The `VAULT_TOKEN` de-identification type is supported only for `TXT`, `CSV`, `JSON`, `JSONL`, `XML`, `DOCX`, `XLSX` and `PPTX`. @@ -1834,7 +1834,7 @@ switch (response.getStatus()) { } ``` -Outputs are written only when polling reached `SUCCESS`, the request was built from a local file, and the run delivered them as base64. A request built from a `SKYFLOW_ID` or `PRESIGNED_URL` has no local name to derive from, and presigned-URL outputs are not fetched, so in those cases read `getOutput()` and write the files yourself. Files are named `processed-.`, with the entities output as `processed-entities-.json` (and `processed-object-entities-.json` for object entities); `outputDirectory` is created if missing and defaults to the working directory. Polling blocks the calling thread. +Outputs are written only when polling reached `SUCCESS`, the request was built from a local file, and the run delivered them as base64. A request built from a `SKYFLOW_ID` or `PRESIGNED_URL` value has no local name to derive from, and presigned-URL outputs are not fetched, so in those cases read `getOutput()` and write the files yourself. Files are named `processed-.`, with the entities output as `processed-entities-.json` (and `processed-object-entities-.json` for object entities); `outputDirectory` is created if missing and defaults to the working directory. Polling blocks the calling thread. ### Polling from a worker thread @@ -1924,18 +1924,18 @@ System.out.println(response.getProcessedText()); // My name is John Doe and my email is j*******@acme.com. ``` -A `RedactionLevel` names one source and one replacement. On skyvault vaults the source is an `entityName` and the replacement a `redactionType`, the same `com.skyflow.enums.RedactionType` that `get` and `detokenize` use. The `tokenGroupName` and `redactionPattern` fields are the flowvault equivalents and do not apply to skyvault vaults. Every field is optional on the API, so the SDK passes the list through as given and the server rejects invalid combinations. +A `RedactionLevel` names one source and one replacement. On skyvault vaults the source is an `entityName` and the replacement a `redactionType`, the same `com.skyflow.enums.RedactionType` that `get` and `detokenize` use. The skyvault SDK's `RedactionLevel` has only these two fields; the flowvault SDK's has `tokenGroupName` and `redactionPattern` instead. Both are optional on the API and are sent as given. ## Reidentify file -`reidentifyFile` is synchronous: the re-identified file comes back in the response, not through a run id. `ReidentifyFileRequest` takes a `dataSource` (`BASE64`, `SKYFLOW_ID` or `PRESIGNED_URL`), the matching `value`, an optional `dataFormat` and the same optional `RedactionLevel` rules as `reidentifyString`. `dataFormat` is limited to the formats that support `VAULT_TOKEN`: `TXT`, `CSV`, `JSON`, `JSONL`, `XML`, `DOCX`, `XLSX` and `PPTX`. +`reidentifyFile` is synchronous: the re-identified file comes back in the response, not through a run id. `ReidentifyFileRequest` takes a `dataSource` (`ReidentifySourceType.BASE64` or `PRESIGNED_URL`; re-identification does not accept a skyflow id as the source), the matching `value`, an optional `dataFormat` and the same optional `RedactionLevel` rules as `reidentifyString`. `dataFormat` is limited to the formats that support `VAULT_TOKEN`: `TXT`, `CSV`, `JSON`, `JSONL`, `XML`, `DOCX`, `XLSX` and `PPTX`. ```java byte[] content = Files.readAllBytes(Paths.get("tokenized.txt")); ReidentifyFileResponse response = skyflowClient.detect().reidentifyFile( ReidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64) + .dataSource(ReidentifySourceType.BASE64) .value(Base64.getEncoder().encodeToString(content)) .dataFormat(FileDataFormat.TXT) .redactionLevel(Collections.singletonList( diff --git a/skyvault/api-report/skyflow-java.baseline.jar b/skyvault/api-report/skyflow-java.baseline.jar index afff0c48..70c1ea81 100644 Binary files a/skyvault/api-report/skyflow-java.baseline.jar and b/skyvault/api-report/skyflow-java.baseline.jar differ diff --git a/skyvault/samples/src/main/java/com/example/detect/GetRunSample.java b/skyvault/samples/src/main/java/com/example/detect/GetRunSample.java index 9d704f25..1d521d71 100644 --- a/skyvault/samples/src/main/java/com/example/detect/GetRunSample.java +++ b/skyvault/samples/src/main/java/com/example/detect/GetRunSample.java @@ -3,7 +3,7 @@ import com.skyflow.Skyflow; import com.skyflow.config.Credentials; import com.skyflow.config.VaultConfig; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.DeidentifySourceType; import com.skyflow.detect.DetectRunStatus; import com.skyflow.detect.FileOutput; import com.skyflow.detect.GetRunRequest; @@ -46,7 +46,7 @@ public static void main(String[] args) throws Exception { System.out.println("Run " + run.getRunId() + " status: " + run.getStatus()); if (run.getStatus() == DetectRunStatus.SUCCESS) { for (FileOutput output : run.getOutput()) { - if (run.getOutputType() != DataSourceType.BASE64) { + if (run.getOutputType() != DeidentifySourceType.BASE64) { // PRESIGNED_URL outputs carry a URL, not base64: download them instead of decoding. System.out.println(output.getProcessedFileType() + " available at " + output.getProcessedFile()); continue; diff --git a/skyvault/samples/src/main/java/com/example/detect/ReidentifyFileSample.java b/skyvault/samples/src/main/java/com/example/detect/ReidentifyFileSample.java index 41c9d765..787d6ef1 100644 --- a/skyvault/samples/src/main/java/com/example/detect/ReidentifyFileSample.java +++ b/skyvault/samples/src/main/java/com/example/detect/ReidentifyFileSample.java @@ -3,7 +3,7 @@ import com.skyflow.Skyflow; import com.skyflow.config.Credentials; import com.skyflow.config.VaultConfig; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.ReidentifySourceType; import com.skyflow.detect.DetectRunStatus; import com.skyflow.detect.EntityType; import com.skyflow.detect.FileDataFormat; @@ -47,7 +47,7 @@ public static void main(String[] args) throws Exception { try { ReidentifyFileResponse response = skyflowClient.detect().reidentifyFile( ReidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64) + .dataSource(ReidentifySourceType.BASE64) .value(base64) .dataFormat(FileDataFormat.TXT) // Optional: render one entity type with a redaction type instead of its plain value. @@ -63,7 +63,7 @@ public static void main(String[] args) throws Exception { int index = 0; for (FileOutput output : response.getOutput()) { index++; - if (response.getOutputType() != DataSourceType.BASE64) { + if (response.getOutputType() != ReidentifySourceType.BASE64) { // PRESIGNED_URL outputs carry a URL, not base64: download them instead of decoding. System.out.println(output.getProcessedFileType() + " available at " + output.getProcessedFile()); continue; diff --git a/skyvault/src/main/java/com/skyflow/VaultClient.java b/skyvault/src/main/java/com/skyflow/VaultClient.java index 99301026..ee55d0ff 100644 --- a/skyvault/src/main/java/com/skyflow/VaultClient.java +++ b/skyvault/src/main/java/com/skyflow/VaultClient.java @@ -235,7 +235,7 @@ private void updateVaultURL() { private void updateExecutorInHTTP() { if (sharedHttpClient == null) { - sharedHttpClient = buildSharedHttpClient(() -> this.token); + sharedHttpClient = buildSharedHttpClient(this::getToken); apiClientBuilder.httpClient(sharedHttpClient); } } diff --git a/skyvault/src/main/java/com/skyflow/detect/DeidentifySourceType.java b/skyvault/src/main/java/com/skyflow/detect/DeidentifySourceType.java new file mode 100644 index 00000000..1649da53 --- /dev/null +++ b/skyvault/src/main/java/com/skyflow/detect/DeidentifySourceType.java @@ -0,0 +1,11 @@ +package com.skyflow.detect; + +/** + * How the input file is supplied to {@code deidentifyFile} on a skyvault vault: base64 content, the skyflow id of a file record already in the vault, or an S3 presigned URL. + * Also the {@code getOutputType()} of {@code DeidentifyFileResponse} and {@code GetRunResponse}. + */ +public enum DeidentifySourceType implements DeidentifyFileSource { + BASE64, + SKYFLOW_ID, + PRESIGNED_URL; +} diff --git a/skyvault/src/main/java/com/skyflow/detect/RedactionLevel.java b/skyvault/src/main/java/com/skyflow/detect/RedactionLevel.java new file mode 100644 index 00000000..184a2945 --- /dev/null +++ b/skyvault/src/main/java/com/skyflow/detect/RedactionLevel.java @@ -0,0 +1,99 @@ +package com.skyflow.detect; + +import com.skyflow.enums.RedactionType; + +/** + * Controls how one group of tokens is rendered when re-identifying on a skyvault vault: the {@code entityName} + * the rule applies to and the {@code redactionType} to render it with ({@code PLAIN_TEXT}, {@code MASKED}, + * {@code REDACTED} or {@code DEFAULT}). Element of {@code redactionLevel} on {@code ReidentifyStringRequest} and + * {@code ReidentifyFileRequest}. Both fields are optional on the API and are sent as given. + */ +public final class RedactionLevel extends RedactionRule { + private final EntityType entityName; + private final RedactionType redactionType; + + private RedactionLevel(RedactionLevelBuilder builder) { + this.entityName = builder.entityName; + this.redactionType = builder.redactionType; + } + + /** + * @return a new builder + */ + public static RedactionLevelBuilder builder() { + return new RedactionLevelBuilder(); + } + + /** + * @return Entity type the rule applies to. + */ + public EntityType getEntityName() { + return entityName; + } + + /** + * @return Rendering applied to that entity's values. + */ + public RedactionType getRedactionType() { + return redactionType; + } + + @Override + String apiEntityName() { + return entityName == null ? null : entityName.name(); + } + + @Override + String apiRedactionType() { + return redactionType == null ? null : redactionType.name(); + } + + @Override + String apiTokenGroupName() { + return null; + } + + @Override + String apiRedactionPattern() { + return null; + } + + @Override + public String toString() { + return "RedactionLevel{" + + "entityName=" + entityName + + ", redactionType=" + redactionType + + '}'; + } + + public static final class RedactionLevelBuilder { + private EntityType entityName; + private RedactionType redactionType; + + private RedactionLevelBuilder() { + } + + /** + * @param entityName Entity type the rule applies to. + * @return this builder + */ + public RedactionLevelBuilder entityName(EntityType entityName) { + this.entityName = entityName; + return this; + } + + /** + * @param redactionType Rendering applied to that entity's values, the same {@code com.skyflow.enums.RedactionType} + * used by {@code get} and {@code detokenize}. + * @return this builder + */ + public RedactionLevelBuilder redactionType(RedactionType redactionType) { + this.redactionType = redactionType; + return this; + } + + public RedactionLevel build() { + return new RedactionLevel(this); + } + } +} diff --git a/skyvault/src/main/java/com/skyflow/detect/ReidentifySourceType.java b/skyvault/src/main/java/com/skyflow/detect/ReidentifySourceType.java new file mode 100644 index 00000000..df8e5bda --- /dev/null +++ b/skyvault/src/main/java/com/skyflow/detect/ReidentifySourceType.java @@ -0,0 +1,11 @@ +package com.skyflow.detect; + +/** + * How the input file is supplied to {@code reidentifyFile} on a skyvault vault: base64 content or an S3 presigned + * URL. Re-identification does not accept a skyflow id as the source. Also the {@code getOutputType()} of + * {@code ReidentifyFileResponse}. + */ +public enum ReidentifySourceType implements ReidentifyFileSource { + BASE64, + PRESIGNED_URL; +} diff --git a/skyvault/src/main/java/com/skyflow/vault/controller/DetectController.java b/skyvault/src/main/java/com/skyflow/vault/controller/DetectController.java index c78fb6ba..3c1fad5f 100644 --- a/skyvault/src/main/java/com/skyflow/vault/controller/DetectController.java +++ b/skyvault/src/main/java/com/skyflow/vault/controller/DetectController.java @@ -2,6 +2,10 @@ import com.skyflow.config.Credentials; import com.skyflow.config.VaultConfig; +import com.skyflow.detect.DeidentifyFileSource; +import com.skyflow.detect.DeidentifySourceType; +import com.skyflow.detect.ReidentifyFileSource; +import com.skyflow.detect.ReidentifySourceType; import com.skyflow.errors.SkyflowException; import com.skyflow.utils.Utils; @@ -36,6 +40,16 @@ protected String resolveVaultUrl() throws SkyflowException { return Utils.getVaultURL(getVaultConfig().getClusterId(), getVaultConfig().getEnv()); } + @Override + protected DeidentifyFileSource toDeidentifySource(String wireValue) { + return enumOrNull(DeidentifySourceType.class, wireValue); + } + + @Override + protected ReidentifyFileSource toReidentifySource(String wireValue) { + return enumOrNull(ReidentifySourceType.class, wireValue); + } + @Override protected String getSdkMetrics() { return Utils.getMetrics().toString(); diff --git a/skyvault/src/test/java/com/skyflow/AuthInterceptorTests.java b/skyvault/src/test/java/com/skyflow/AuthInterceptorTests.java new file mode 100644 index 00000000..562a847d --- /dev/null +++ b/skyvault/src/test/java/com/skyflow/AuthInterceptorTests.java @@ -0,0 +1,183 @@ +package com.skyflow; + +import com.skyflow.config.BaseCredentials; +import com.skyflow.config.Credentials; +import com.skyflow.config.VaultConfig; +import com.skyflow.enums.Env; +import com.skyflow.errors.SkyflowException; +import com.skyflow.utils.FakeChain; +import okhttp3.Interceptor; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Base64; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * The auth interceptor installed by skyvault's {@code VaultClient.updateExecutorInHTTP()}. + * + *

Skyvault's shared OkHttp client carries this one interceptor; the generated {@code ClientOptions} + * adds no transport retry when a client is supplied. The refresh-and-resend logic lives in + * {@code BaseVaultClient} and is unit tested there; these tests prove the skyvault client wires it + * in with {@code this::getToken}, so a token refreshed after a {@code 401} reaches the resend. + * Mirrors flowvault's {@code AuthInterceptorTests}. + */ +public class AuthInterceptorTests { + + private static VaultConfig config() { + VaultConfig config = new VaultConfig(); + config.setVaultId("vault1"); + config.setClusterId("cluster1"); + config.setEnv(Env.DEV); + return config; + } + + private static VaultConfig configWithCredentialsString() { + Credentials credentials = new Credentials(); + credentials.setCredentialsString("{\"clientID\":\"c\"}"); + VaultConfig config = config(); + config.setCredentials(credentials); + return config; + } + + private static String jwtExpiringIn(long seconds) { + Base64.Encoder enc = Base64.getUrlEncoder().withoutPadding(); + String header = enc.encodeToString("{\"alg\":\"none\"}".getBytes(StandardCharsets.UTF_8)); + String payload = enc.encodeToString(("{\"exp\":" + (System.currentTimeMillis() / 1000 + seconds) + "}") + .getBytes(StandardCharsets.UTF_8)); + return header + "." + payload + ".sig"; + } + + /** Skyvault client with mintable credentials and a scripted token source. */ + private static final class MintingClient extends VaultClient { + final AtomicInteger generations = new AtomicInteger(); + final String fresh; + + MintingClient(String fresh) { + super(configWithCredentialsString(), null); + this.fresh = fresh; + this.finalCredentials = getVaultConfig().getCredentials(); + } + + @Override + protected String generateBearerToken(BaseCredentials credentials) { + generations.incrementAndGet(); + return fresh; + } + } + + /** updateExecutorInHTTP is private in skyvault; install the client the way setBearerToken does. */ + private static Interceptor authInterceptorOf(VaultClient client) throws Exception { + Method install = VaultClient.class.getDeclaredMethod("updateExecutorInHTTP"); + install.setAccessible(true); + install.invoke(client); + List interceptors = client.sharedHttpClient.interceptors(); + Assert.assertEquals("skyvault's shared client carries the auth interceptor alone", 1, interceptors.size()); + return interceptors.get(0); + } + + @Test + public void testAuthInterceptor_regeneratesOnceAndResendsOn401ForExpiredToken() throws Exception { + MintingClient client = new MintingClient("fresh-token"); + client.token = jwtExpiringIn(-1); + FakeChain chain = new FakeChain(401, 200); + + Assert.assertEquals(200, authInterceptorOf(client).intercept(chain).code()); + Assert.assertEquals(2, chain.calls()); + Assert.assertEquals("Bearer fresh-token", chain.lastProceeded().header("Authorization")); + Assert.assertEquals("the cache must hold the new token for the calls that follow", "fresh-token", client.token); + Assert.assertEquals(1, client.generations.get()); + Assert.assertTrue("the rejected response must be closed before the resend", chain.bodies().get(0).isClosed()); + } + + @Test + public void testAuthInterceptor_401ForTokenWithLifeLeftIsReturnedWithoutRegenerating() throws Exception { + MintingClient client = new MintingClient("fresh-token"); + String live = jwtExpiringIn(900); + client.token = live; + FakeChain chain = new FakeChain(401, 200); + + Assert.assertEquals(401, authInterceptorOf(client).intercept(chain).code()); + Assert.assertEquals(1, chain.calls()); + Assert.assertEquals(0, client.generations.get()); + Assert.assertEquals(live, client.token); + } + + @Test + public void testAuthInterceptor_401WithStaticTokenIsReturnedAsToday() throws Exception { + Credentials credentials = new Credentials(); + credentials.setToken(jwtExpiringIn(-1)); + VaultConfig config = config(); + config.setCredentials(credentials); + VaultClient client = new VaultClient(config, null); + client.setBearerToken(); + FakeChain chain = new FakeChain(401, 200); + + Assert.assertEquals(401, authInterceptorOf(client).intercept(chain).code()); + Assert.assertEquals(1, chain.calls()); + } + + @Test + public void testAuthInterceptor_401WithApiKeyIsReturnedAsToday() throws Exception { + // Loaded from dummy-non-secrets/ (Gitleaks-excluded) rather than a string literal, + // since an API-key-shaped literal would itself trip the scanner. + String apiKey = new String(Files.readAllBytes( + Paths.get("./src/test/resources/dummy-non-secrets/dummy-api-key.txt")), StandardCharsets.UTF_8).trim(); + Credentials credentials = new Credentials(); + credentials.setApiKey(apiKey); + VaultConfig config = config(); + config.setCredentials(credentials); + VaultClient client = new VaultClient(config, null); + client.setBearerToken(); + FakeChain chain = new FakeChain(401, 200); + + Assert.assertEquals(401, authInterceptorOf(client).intercept(chain).code()); + Assert.assertEquals(1, chain.calls()); + Assert.assertEquals("Bearer " + apiKey, chain.lastProceeded().header("Authorization")); + } + + @Test + public void testAuthInterceptor_resendThatAlsoGets401IsNotRetriedAgain() throws Exception { + MintingClient client = new MintingClient("fresh-token"); + client.token = jwtExpiringIn(-1); + FakeChain chain = new FakeChain(401, 401, 200); + + Assert.assertEquals(401, authInterceptorOf(client).intercept(chain).code()); + Assert.assertEquals(2, chain.calls()); + Assert.assertEquals(1, client.generations.get()); + } + + @Test + public void testAuthInterceptor_readsTheRefreshedTokenOnTheResend() throws Exception { + // setBearerToken() is what production code calls; it must install the same interceptor and + // the resend must carry the token that refresh stored, read back through getToken(). + MintingClient client = new MintingClient("fresh-token"); + client.setBearerToken(); + Assert.assertEquals("fresh-token", client.token); + client.token = jwtExpiringIn(-1); + FakeChain chain = new FakeChain(401, 200); + + Interceptor interceptor = client.sharedHttpClient.interceptors().get(0); + Assert.assertEquals(200, interceptor.intercept(chain).code()); + Assert.assertEquals("Bearer fresh-token", chain.lastProceeded().header("Authorization")); + } + + @Test + public void testAuthInterceptor_isTheOnlyInterceptorOnTheSharedClient() throws SkyflowException { + Credentials credentials = new Credentials(); + credentials.setToken(jwtExpiringIn(3600)); + VaultConfig config = config(); + config.setCredentials(credentials); + VaultClient client = new VaultClient(config, null); + client.setBearerToken(); + + Assert.assertEquals("no transport retry wraps auth in skyvault; a 401 reaches auth directly", + 1, client.sharedHttpClient.interceptors().size()); + } +} diff --git a/skyvault/src/test/java/com/skyflow/detect/DetectSourceTypesTests.java b/skyvault/src/test/java/com/skyflow/detect/DetectSourceTypesTests.java new file mode 100644 index 00000000..65ce17c0 --- /dev/null +++ b/skyvault/src/test/java/com/skyflow/detect/DetectSourceTypesTests.java @@ -0,0 +1,39 @@ +package com.skyflow.detect; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class DetectSourceTypesTests { + + private static List names(Enum[] values) { + List names = new ArrayList<>(); + for (Enum value : values) { + names.add(value.name()); + } + return names; + } + + @Test + public void deidentifySourcesOfferedOnSkyvault() { + Assert.assertEquals(Arrays.asList("BASE64", "SKYFLOW_ID", "PRESIGNED_URL"), names(DeidentifySourceType.values())); + DeidentifyFileSource asSource = DeidentifySourceType.BASE64; + Assert.assertEquals("BASE64", asSource.name()); + } + + @Test + public void reidentifySourcesNeverIncludeSkyflowId() { + Assert.assertEquals(Arrays.asList("BASE64", "PRESIGNED_URL"), names(ReidentifySourceType.values())); + try { + ReidentifySourceType.valueOf("SKYFLOW_ID"); + Assert.fail("re-identification does not accept a skyflow id"); + } catch (IllegalArgumentException expected) { + // not offered + } + ReidentifyFileSource asSource = ReidentifySourceType.PRESIGNED_URL; + Assert.assertEquals("PRESIGNED_URL", asSource.name()); + } +} diff --git a/skyvault/src/test/java/com/skyflow/detect/RedactionLevelTests.java b/skyvault/src/test/java/com/skyflow/detect/RedactionLevelTests.java new file mode 100644 index 00000000..b32f58d5 --- /dev/null +++ b/skyvault/src/test/java/com/skyflow/detect/RedactionLevelTests.java @@ -0,0 +1,28 @@ +package com.skyflow.detect; + +import com.skyflow.enums.RedactionType; +import org.junit.Assert; +import org.junit.Test; + +public class RedactionLevelTests { + + @Test + public void skyvaultRuleCarriesEntityNameAndRedactionTypeOnly() { + RedactionLevel rule = RedactionLevel.builder().entityName(EntityType.NAME).redactionType(RedactionType.MASKED).build(); + Assert.assertEquals(EntityType.NAME, rule.getEntityName()); + Assert.assertEquals(RedactionType.MASKED, rule.getRedactionType()); + + com.skyflow.generated.common.rest.types.RedactionLevel api = RedactionRules.toApi(rule); + Assert.assertEquals("NAME", api.getEntityName().get().toString()); + Assert.assertEquals("MASKED", api.getRedactionType().get().toString()); + Assert.assertFalse(api.getTokenGroupName().isPresent()); + Assert.assertFalse(api.getRedactionPattern().isPresent()); + } + + @Test + public void unsetFieldsAreNotSent() { + com.skyflow.generated.common.rest.types.RedactionLevel api = RedactionRules.toApi(RedactionLevel.builder().build()); + Assert.assertFalse(api.getEntityName().isPresent()); + Assert.assertFalse(api.getRedactionType().isPresent()); + } +} diff --git a/skyvault/src/test/java/com/skyflow/utils/FakeChain.java b/skyvault/src/test/java/com/skyflow/utils/FakeChain.java new file mode 100644 index 00000000..8945788a --- /dev/null +++ b/skyvault/src/test/java/com/skyflow/utils/FakeChain.java @@ -0,0 +1,140 @@ +package com.skyflow.utils; + +import okhttp3.Call; +import okhttp3.Connection; +import okhttp3.Interceptor; +import okhttp3.MediaType; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okio.Buffer; +import okio.BufferedSource; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Minimal Interceptor.Chain that replays a scripted list of status codes, so the retry loop can + * be driven without a network or a mock-server dependency. Records how many times it was called + * and which responses were closed. + */ +public final class FakeChain implements Interceptor.Chain { + + private final Request request = + new Request.Builder().url("https://cluster1.vault.skyflowapis.dev/v1/vaults/vault1/table1").build(); + private final int[] statusCodes; + private final List bodies = new ArrayList<>(); + private int calls; + private Request lastProceeded; + + public FakeChain(int... statusCodes) { + this.statusCodes = statusCodes; + } + + public int calls() { + return calls; + } + + /** Responses the interceptor superseded and should have closed. */ + public List bodies() { + return bodies; + } + + /** The request as it reached the next interceptor, i.e. after any rewriting. */ + public Request lastProceeded() { + return lastProceeded; + } + + @Override + public Request request() { + return request; + } + + @Override + public Response proceed(Request request) throws IOException { + this.lastProceeded = request; + // Past the end of the script, keep returning the last code. + int code = statusCodes[Math.min(calls, statusCodes.length - 1)]; + calls++; + TrackingBody body = new TrackingBody(); + bodies.add(body); + return new Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(code) + .message("status " + code) + .body(body) + .build(); + } + + @Override + public Connection connection() { + return null; + } + + @Override + public Call call() { + throw new UnsupportedOperationException(); + } + + @Override + public int connectTimeoutMillis() { + return 0; + } + + @Override + public Interceptor.Chain withConnectTimeout(int timeout, TimeUnit unit) { + return this; + } + + @Override + public int readTimeoutMillis() { + return 0; + } + + @Override + public Interceptor.Chain withReadTimeout(int timeout, TimeUnit unit) { + return this; + } + + @Override + public int writeTimeoutMillis() { + return 0; + } + + @Override + public Interceptor.Chain withWriteTimeout(int timeout, TimeUnit unit) { + return this; + } + + public static final class TrackingBody extends ResponseBody { + boolean closed; + + @Override + public MediaType contentType() { + return null; + } + + @Override + public long contentLength() { + return 0; + } + + @Override + public BufferedSource source() { + return new Buffer(); + } + + @Override + public void close() { + closed = true; + } + + public boolean isClosed() { + return closed; + } + } +} diff --git a/skyvault/src/test/java/com/skyflow/utils/UtilsTests.java b/skyvault/src/test/java/com/skyflow/utils/UtilsTests.java index 7ac13ec7..180d417b 100644 --- a/skyvault/src/test/java/com/skyflow/utils/UtilsTests.java +++ b/skyvault/src/test/java/com/skyflow/utils/UtilsTests.java @@ -1,15 +1,16 @@ package com.skyflow.utils; -import com.google.gson.JsonObject; -import com.skyflow.config.ConnectionConfig; -import com.skyflow.enums.RequestMethod; -import com.skyflow.vault.connection.InvokeConnectionRequest; +import java.util.HashMap; +import java.util.Map; + import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import java.util.HashMap; -import java.util.Map; +import com.google.gson.JsonObject; +import com.skyflow.config.ConnectionConfig; +import com.skyflow.enums.RequestMethod; +import com.skyflow.vault.connection.InvokeConnectionRequest; public class UtilsTests { private static final String INVALID_EXCEPTION_THROWN = "Should not have thrown any exception"; @@ -94,13 +95,13 @@ public void testGetMetricsWithException() { } catch (Exception e) { Assert.fail(INVALID_EXCEPTION_THROWN); } finally { - restoreProperty("os.name", osName); - restoreProperty("os.version", osVersion); - restoreProperty("java.version", javaVersion); + restore("os.name", osName); + restore("os.version", osVersion); + restore("java.version", javaVersion); } } - private static void restoreProperty(String key, String value) { + private static void restore(String key, String value) { if (value == null) { System.clearProperty(key); } else { diff --git a/skyvault/src/test/java/com/skyflow/vault/controller/DetectControllerTests.java b/skyvault/src/test/java/com/skyflow/vault/controller/DetectControllerTests.java index b31002ef..ddc5e7cc 100644 --- a/skyvault/src/test/java/com/skyflow/vault/controller/DetectControllerTests.java +++ b/skyvault/src/test/java/com/skyflow/vault/controller/DetectControllerTests.java @@ -1,11 +1,12 @@ package com.skyflow.vault.controller; +import com.skyflow.detect.ReidentifySourceType; import com.skyflow.Skyflow; import com.skyflow.config.Credentials; import com.skyflow.config.VaultConfig; import com.skyflow.detect.CheckGuardrailsRequest; import com.skyflow.detect.CheckGuardrailsResponse; -import com.skyflow.detect.DataSourceType; +import com.skyflow.detect.DeidentifySourceType; import com.skyflow.detect.DeidentificationType; import com.skyflow.detect.DeidentifyFileRequest; import com.skyflow.detect.DeidentifyFileResponse; @@ -22,6 +23,7 @@ import com.skyflow.detect.GuardrailsValidation; import com.skyflow.detect.PollOptions; import com.skyflow.detect.RedactionLevel; +import com.skyflow.enums.RedactionType; import com.skyflow.detect.ReidentifyFileRequest; import com.skyflow.detect.ReidentifyFileResponse; import com.skyflow.detect.ReidentifyStringRequest; @@ -242,7 +244,7 @@ public void reidentifyStringRoundTrip() throws Exception { ReidentifyStringResponse response = controllerFor(server).reidentifyString(ReidentifyStringRequest.builder() .text("My name is [NAME_1].") .redactionLevel(Collections.singletonList( - RedactionLevel.builder().tokenGroupName("names").redactionPattern("mask").build())) + RedactionLevel.builder().entityName(EntityType.NAME).redactionType(RedactionType.MASKED).build())) .build()); Assert.assertEquals("My name is J**n D*e.", response.getProcessedText()); @@ -253,8 +255,8 @@ public void reidentifyStringRoundTrip() throws Exception { String body = recorded.getBody().readUtf8(); Assert.assertTrue(body, body.contains("\"text\":\"My name is [NAME_1].\"")); Assert.assertTrue(body, body.contains("\"vaultId\":\"vault-1\"")); - Assert.assertTrue(body, body.contains("\"tokenGroupName\":\"names\"")); - Assert.assertTrue(body, body.contains("\"redactionPattern\":\"mask\"")); + Assert.assertTrue(body, body.contains("\"entityName\":\"NAME\"")); + Assert.assertTrue(body, body.contains("\"redactionType\":\"MASKED\"")); } finally { server.shutdown(); } @@ -276,7 +278,7 @@ public void getRunRoundTripQueriesByIdScopedToVault() throws Exception { GetRunResponse done = controller.getRun(GetRunRequest.builder().runId(" run-42 ").build()); Assert.assertEquals("run-42", done.getRunId()); Assert.assertEquals(DetectRunStatus.SUCCESS, done.getStatus()); - Assert.assertEquals(DataSourceType.BASE64, done.getOutputType()); + Assert.assertEquals(DeidentifySourceType.BASE64, done.getOutputType()); Assert.assertEquals("UERG", done.getOutput().get(0).getProcessedFile()); Assert.assertEquals("REDACTED_FILE", done.getOutput().get(0).getProcessedFileType()); Assert.assertEquals(FileDataFormat.PDF, done.getOutput().get(0).getProcessedFileExtension()); @@ -365,7 +367,7 @@ public void reidentifyFileValidatesBeforeResolvingCredentials() throws SkyflowEx ReidentifyFileRequest[] bad = { null, ReidentifyFileRequest.builder().value("Zm9v").build(), - ReidentifyFileRequest.builder().dataSource(DataSourceType.BASE64).value(" ").build(), + ReidentifyFileRequest.builder().dataSource(ReidentifySourceType.BASE64).value(" ").build(), }; BaseErrorMessage[] expected = { BaseErrorMessage.ReidentifyFileRequestNull, @@ -388,7 +390,7 @@ public void reidentifyFileMissingCredentialsSurfaceWithoutNetworkCall() throws S DetectController controller = new DetectController(vaultConfig("vault-a"), emptyCredentials()); try { controller.reidentifyFile(ReidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64).value("Zm9v").build()); + .dataSource(ReidentifySourceType.BASE64).value("Zm9v").build()); Assert.fail("expected SkyflowException"); } catch (SkyflowException e) { Assert.assertEquals(ErrorCode.INVALID_INPUT.getCode(), e.getHttpCode()); @@ -408,15 +410,15 @@ public void reidentifyFileSendsScopedRequestAndMapsResponse() throws Exception { + "\"metrics\":{\"size\":0.5,\"wordCount\":3,\"characterCount\":12}}")); ReidentifyFileResponse response = controllerFor(server).reidentifyFile(ReidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64) + .dataSource(ReidentifySourceType.BASE64) .value("W05BTUVfMV0=") .dataFormat(FileDataFormat.TXT) .redactionLevel(Arrays.asList( - RedactionLevel.builder().tokenGroupName("names").redactionPattern("mask").build())) + RedactionLevel.builder().entityName(EntityType.NAME).redactionType(RedactionType.MASKED).build())) .build()); Assert.assertEquals(DetectRunStatus.SUCCESS, response.getStatus()); - Assert.assertEquals(DataSourceType.BASE64, response.getOutputType()); + Assert.assertEquals(ReidentifySourceType.BASE64, response.getOutputType()); Assert.assertEquals(1, response.getOutput().size()); Assert.assertEquals("SGVsbG8=", response.getOutput().get(0).getProcessedFile()); Assert.assertEquals("REIDENTIFIED_FILE", response.getOutput().get(0).getProcessedFileType()); @@ -433,8 +435,8 @@ public void reidentifyFileSendsScopedRequestAndMapsResponse() throws Exception { Assert.assertTrue(body, body.contains("\"value\":\"W05BTUVfMV0=\"")); Assert.assertTrue(body, body.contains("\"vaultId\":\"vault-1\"")); Assert.assertTrue(body, body.contains("\"dataFormat\":\"txt\"")); - Assert.assertTrue(body, body.contains("\"tokenGroupName\":\"names\"")); - Assert.assertTrue(body, body.contains("\"redactionPattern\":\"mask\"")); + Assert.assertTrue(body, body.contains("\"entityName\":\"NAME\"")); + Assert.assertTrue(body, body.contains("\"redactionType\":\"MASKED\"")); } finally { server.shutdown(); } @@ -447,7 +449,7 @@ public void reidentifyFileToleratesSparseAndUnknownResponseValues() throws Excep try { server.enqueue(new MockResponse().setResponseCode(200).setBody("{\"status\":\"SOMETHING_NEW\"}")); ReidentifyFileResponse response = controllerFor(server).reidentifyFile(ReidentifyFileRequest.builder() - .dataSource(DataSourceType.SKYFLOW_ID).value("rec-1").build()); + .dataSource(ReidentifySourceType.PRESIGNED_URL).value("https://example.com/tokenized.txt").build()); Assert.assertEquals(DetectRunStatus.UNKNOWN, response.getStatus()); Assert.assertNull(response.getOutputType()); Assert.assertTrue(response.getOutput().isEmpty()); @@ -470,7 +472,7 @@ public void reidentifyFileMapsApiErrors() throws Exception { + "\"http_status\":\"Bad Request\",\"details\":[]}}")); try { controllerFor(server).reidentifyFile(ReidentifyFileRequest.builder() - .dataSource(DataSourceType.PRESIGNED_URL).value("https://example.com/f").build()); + .dataSource(ReidentifySourceType.PRESIGNED_URL).value("https://example.com/f").build()); Assert.fail("expected SkyflowException"); } catch (SkyflowException e) { Assert.assertEquals(400, e.getHttpCode()); @@ -505,7 +507,7 @@ public void deidentifyFileSubmitOnlyReturnsRunIdWithoutPolling() throws Exceptio server.enqueue(new MockResponse().setResponseCode(200).setBody("{\"runId\":\"run-42\"}")); DeidentifyFileResponse response = controllerFor(server).deidentifyFile(DeidentifyFileRequest.builder() - .dataSource(DataSourceType.SKYFLOW_ID).value("rec-1").dataFormat(FileDataFormat.PDF) + .dataSource(DeidentifySourceType.SKYFLOW_ID).value("rec-1").dataFormat(FileDataFormat.PDF) .configurationId("cfg-1").build()); Assert.assertEquals("run-42", response.getRunId()); @@ -550,7 +552,7 @@ public void deidentifyFilePollsToSuccessAndWritesOutputsForLocalFile() throws Ex Assert.assertEquals("run-7", response.getRunId()); Assert.assertEquals(DetectRunStatus.SUCCESS, response.getStatus()); - Assert.assertEquals(DataSourceType.BASE64, response.getOutputType()); + Assert.assertEquals(DeidentifySourceType.BASE64, response.getOutputType()); Assert.assertEquals(2, response.getOutput().size()); Assert.assertEquals(Integer.valueOf(3), response.getMetrics().getWordCount()); Assert.assertEquals("Pay [NAME_1]", new String(Files.readAllBytes(new File(outDir, "processed-invoice.txt").toPath()), "UTF-8")); @@ -578,7 +580,7 @@ public void deidentifyFileReturnsRunIdWithStatusWhenBudgetRunsOut() throws Excep server.enqueue(new MockResponse().setResponseCode(200).setBody("{\"status\":\"IN_PROGRESS\"}")); DeidentifyFileResponse response = controllerFor(server).deidentifyFile(DeidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64).value("Zm9v") + .dataSource(DeidentifySourceType.BASE64).value("Zm9v") .pollOptions(PollOptions.builder().maxAttempts(1).build()).build()); Assert.assertEquals("run-8", response.getRunId()); @@ -599,7 +601,7 @@ public void deidentifyFileSurfacesFailedRunMessage() throws Exception { server.enqueue(new MockResponse().setResponseCode(200).setBody("{\"status\":\"FAILED\",\"message\":\"unsupported file\"}")); DeidentifyFileResponse response = controllerFor(server).deidentifyFile(DeidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64).value("Zm9v").pollOptions(PollOptions.builder().build()).build()); + .dataSource(DeidentifySourceType.BASE64).value("Zm9v").pollOptions(PollOptions.builder().build()).build()); Assert.assertEquals(DetectRunStatus.FAILED, response.getStatus()); Assert.assertEquals("unsupported file", response.getMessage()); @@ -618,7 +620,7 @@ public void deidentifyFileSubmitErrorMapsToSkyflowException() throws Exception { .setBody("{\"error\":{\"grpc_code\":3,\"http_code\":400,\"message\":\"configuration not found\",\"http_status\":\"Bad Request\",\"details\":[]}}")); try { controllerFor(server).deidentifyFile(DeidentifyFileRequest.builder() - .dataSource(DataSourceType.BASE64).value("Zm9v").configurationId("missing").build()); + .dataSource(DeidentifySourceType.BASE64).value("Zm9v").configurationId("missing").build()); Assert.fail("expected SkyflowException"); } catch (SkyflowException e) { Assert.assertEquals(400, e.getHttpCode());