feat(export): Add client for /export endpoint #574
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| package io.weaviate.integration; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.concurrent.TimeoutException; | ||
|
|
||
| import org.assertj.core.api.Assertions; | ||
| import org.assertj.core.api.InstanceOfAssertFactories; | ||
| import org.junit.BeforeClass; | ||
| import org.junit.Test; | ||
|
|
||
| import io.weaviate.ConcurrentTest; | ||
| import io.weaviate.client6.v1.api.WeaviateClient; | ||
| import io.weaviate.client6.v1.api.export.Export; | ||
| import io.weaviate.client6.v1.api.export.ExportStatus; | ||
| import io.weaviate.client6.v1.api.export.ShardExportProgress; | ||
| import io.weaviate.containers.Weaviate; | ||
|
|
||
| public class ExportITest extends ConcurrentTest { | ||
| private static final WeaviateClient client = Weaviate.custom() | ||
| .withExportPath("/tmp/export").build() | ||
| .getClient(); | ||
|
|
||
| @BeforeClass | ||
| public static void __() { | ||
| Weaviate.Version.V137.orSkip(); | ||
| } | ||
|
|
||
| @Test | ||
| public void test_lifecycle() throws IOException, TimeoutException { | ||
| // Arrange | ||
| String nsA = ns("A"), nsB = ns("B"), nsC = ns("C"); | ||
| String exportId = ns("export_1").toLowerCase(); | ||
| String backend = "filesystem"; | ||
|
|
||
| var collectionA = client.collections.create(nsA); | ||
| var collectionB = client.collections.create(nsB); | ||
| var collectionC = client.collections.create(nsC); | ||
|
|
||
| // Insert some data | ||
| for (var c : List.of(collectionA, collectionB, collectionC)) { | ||
| var resp = c.data.insertMany(Map.of(), Map.of(), Map.of()); | ||
| Assertions.assertThat(resp.errors()).isEmpty(); | ||
| } | ||
|
|
||
| // Act: start export | ||
| var started = client.export.create(exportId, backend, | ||
| export -> export | ||
| .includeCollections(nsA, nsB)); | ||
|
|
||
| // Assert | ||
| Assertions.assertThat(started) | ||
| .as("created export operation") | ||
| .returns(exportId, Export::id) | ||
| .returns(backend, Export::backend) | ||
| .returns(ExportStatus.STARTED, Export::status) | ||
| .returns(null, Export::error) | ||
| .extracting(Export::includesCollections, InstanceOfAssertFactories.list(String.class)) | ||
| .containsOnly(nsA, nsB); | ||
|
|
||
| // Act: await export competion | ||
| var completed = started.waitForCompletion(client); | ||
|
|
||
| // Assert | ||
| Assertions.assertThat(completed) | ||
| .as("await export completion") | ||
| .returns(exportId, Export::id) | ||
| .returns(backend, Export::backend) | ||
| .returns(ExportStatus.SUCCESS, Export::status) | ||
| .returns(null, Export::error) | ||
| .extracting(Export::includesCollections, InstanceOfAssertFactories.list(String.class)) | ||
| .containsOnly(nsA, nsB); | ||
|
|
||
| Assertions.assertThat(completed) | ||
| .extracting(Export::shardStatus, InstanceOfAssertFactories.map(String.class, | ||
| Object.class)) | ||
| .allSatisfy((__, shards) -> { | ||
| Assertions.assertThat(shards) | ||
| .asInstanceOf(InstanceOfAssertFactories.map(String.class, ShardExportProgress.class)) | ||
| .isNotEmpty(); | ||
| }); | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| package io.weaviate.client6.v1.api.export; | ||
|
|
||
| import java.util.Collections; | ||
|
|
||
| import io.weaviate.client6.v1.internal.rest.Endpoint; | ||
| import io.weaviate.client6.v1.internal.rest.SimpleEndpoint; | ||
|
|
||
| public record CancelExportRequest(String exportId, String backend) { | ||
|
|
||
| public static Endpoint<CancelExportRequest, Void> _ENDPOINT = SimpleEndpoint.sideEffect( | ||
| __ -> "DELETE", | ||
| request -> { | ||
| var cancel = (CancelExportRequest) request; | ||
| return "/export/" + cancel.backend + "/" + cancel.exportId; | ||
| }, | ||
| request -> Collections.emptyMap()) | ||
| .allowStatus(409); | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| package io.weaviate.client6.v1.api.export; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.Arrays; | ||
| import java.util.Collections; | ||
| import java.util.List; | ||
| import java.util.function.Function; | ||
|
|
||
| import com.google.gson.annotations.SerializedName; | ||
|
|
||
| import io.weaviate.client6.v1.internal.ObjectBuilder; | ||
| import io.weaviate.client6.v1.internal.json.JSON; | ||
| import io.weaviate.client6.v1.internal.rest.Endpoint; | ||
| import io.weaviate.client6.v1.internal.rest.SimpleEndpoint; | ||
|
|
||
| public record CreateExportRequest(ExportCreate body, String backend) { | ||
|
|
||
| public static Endpoint<CreateExportRequest, Export> _ENDPOINT = new SimpleEndpoint<>( | ||
| request -> "POST", | ||
| request -> "/export/" + request.backend, | ||
| request -> Collections.emptyMap(), | ||
| request -> JSON.serialize(request.body), | ||
| (statusCode, response) -> JSON.deserialize(response, Export.class)); | ||
|
|
||
| public static record ExportCreate( | ||
| @SerializedName("id") String id, | ||
| @SerializedName("file_format") FileFormat fileFormat, | ||
| @SerializedName("include") List<String> includeCollections, | ||
| @SerializedName("exclude") List<String> excludeCollections) { | ||
|
|
||
| public static ExportCreate of(String exportId) { | ||
| return of(exportId, ObjectBuilder.identity()); | ||
| } | ||
|
|
||
| public static ExportCreate of(String exportId, Function<Builder, ObjectBuilder<ExportCreate>> fn) { | ||
| return fn.apply(new Builder(exportId)).build(); | ||
| } | ||
|
|
||
| public ExportCreate(Builder builder) { | ||
| this( | ||
| builder.exportId, | ||
| builder.fileFormat, | ||
| builder.includeCollections, | ||
| builder.excludeCollections); | ||
| } | ||
|
|
||
| public static class Builder implements ObjectBuilder<ExportCreate> { | ||
| private final String exportId; | ||
|
|
||
| private FileFormat fileFormat = FileFormat.PARQUET; | ||
| private final List<String> includeCollections = new ArrayList<>(); | ||
| private final List<String> excludeCollections = new ArrayList<>(); | ||
|
|
||
| public Builder(String exportId) { | ||
| this.exportId = exportId; | ||
| } | ||
|
|
||
| /** Collection that should be included in the backup. */ | ||
| public Builder includeCollections(String... includeCollections) { | ||
| return includeCollections(Arrays.asList(includeCollections)); | ||
| } | ||
|
|
||
| /** Collection that should be included in the backup. */ | ||
| public Builder includeCollections(List<String> includeCollections) { | ||
| this.includeCollections.addAll(includeCollections); | ||
| return this; | ||
| } | ||
|
|
||
| /** Collection that should be excluded from the backup. */ | ||
| public Builder excludeCollections(String... excludeCollections) { | ||
| return excludeCollections(Arrays.asList(excludeCollections)); | ||
| } | ||
|
|
||
| /** Collection that should be excluded from the backup. */ | ||
|
Collaborator
There was a problem hiding this comment. comments say backup in this file |
||
| public Builder excludeCollections(List<String> excludeCollections) { | ||
| this.excludeCollections.addAll(excludeCollections); | ||
| return this; | ||
| } | ||
|
|
||
| /** Export file format. */ | ||
| public Builder fileFormat(FileFormat fileFormat) { | ||
| this.fileFormat = fileFormat; | ||
| return this; | ||
| } | ||
|
|
||
| @Override | ||
| public ExportCreate build() { | ||
| return new ExportCreate(this); | ||
| } | ||
| } | ||
| } | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
204 (accepted) is also ok