Skip to content

POC: Delta rest api #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yili-db wants to merge 1 commit into
base: main
Choose a base branch
from
Open

POC: Delta rest api #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,767 changes: 1,767 additions & 0 deletions api/delta-rest.yaml
View file
Open in desktop

Large diffs are not rendered by default.

111 changes: 105 additions & 6 deletions build.sbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ lazy val server = (project in file("server"))
.dependsOn(client % "test->test")
// Server and control models are added as provided to avoid them being added as maven dependencies
// This is because the server and control models are included in the server jar
.dependsOn(serverModels % "provided", controlModels % "provided")
.dependsOn(serverModels % "provided", controlModels % "provided", deltaRestServerModels % "provided")
.dependsOn(controlApi % "test->compile")
.enablePlugins(CheckstylePlugin)
.settings (
Expand Down Expand Up @@ -413,11 +413,12 @@ lazy val server = (project in file("server"))
(Test / runMain).toTask(s" io.unitycatalog.server.utils.PopulateTestDatabase").value
},
Test / javaOptions += s"-Duser.dir=${(ThisBuild / baseDirectory).value.getAbsolutePath}",
// Include server and control models in the bin package for server
// This will allow us to have a single maven artifact and not 3 (server, server models, control models)
// Include server, control, and delta rest models in the bin package for server
// This will allow us to have a single maven artifact and not multiple (server, server models, control models, deltarest models)
Compile / packageBin / mappings ++= (Compile / packageBin / mappings).value ++
(serverModels / Compile / packageBin / mappings).value ++
(controlModels / Compile / packageBin / mappings).value
(controlModels / Compile / packageBin / mappings).value ++
(deltaRestServerModels / Compile / packageBin / mappings).value
)

lazy val serverModels = (project in file("server") / "target" / "models")
Expand Down Expand Up @@ -492,9 +493,107 @@ lazy val controlModels = (project in file("server") / "target" / "controlmodels"
}
)

lazy val deltaRestServerModels = (project in file("server") / "target" / "deltarestmodels")
.enablePlugins(OpenApiGeneratorPlugin)
.disablePlugins(JavaFormatterPlugin, CheckstylePlugin)
.settings(
name := s"$artifactNamePrefix-deltarestservermodels",
commonSettings,
javaOnlyReleaseSettings,
(Compile / compile) := ((Compile / compile) dependsOn generate).value,
Compile / compile / javacOptions ++= javacRelease17,
libraryDependencies ++= Seq(
"jakarta.annotation" % "jakarta.annotation-api" % "3.0.0" % Provided,
"com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion,
),
// OpenAPI generation configs for generating model codes from the Delta REST Catalog spec
openApiInputSpec := (file(".") / "api" / "delta-rest.yaml").toString,
openApiGeneratorName := "java",
openApiOutputDir := (file("server") / "target" / "deltarestmodels").toString,
openApiValidateSpec := SettingEnabled,
openApiGenerateMetadata := SettingDisabled,
openApiModelPackage := s"$orgName.server.model.deltarest",
openApiAdditionalProperties := Map(
"library" -> "resteasy", // resteasy generates the most minimal models
"useJakartaEe" -> "true",
"hideGenerationTimestamp" -> "true"
),
openApiGlobalProperties := Map("models" -> ""),
openApiGenerateApiTests := SettingDisabled,
openApiGenerateModelTests := SettingDisabled,
openApiGenerateApiDocumentation := SettingDisabled,
openApiGenerateModelDocumentation := SettingDisabled,
// Define the simple generate command to generate model codes
generate := {
val _ = openApiGenerate.value
}
)

lazy val deltaRestClient = (project in file("clients/delta-rest"))
.enablePlugins(OpenApiGeneratorPlugin)
.settings(
name := s"$artifactNamePrefix-deltarest-client",
commonSettings,
javaOnlyReleaseSettings,
Compile / compile / javacOptions ++= javacRelease11,
javaCheckstyleTestOnlySettings("dev/checkstyle-config.xml"),
// Include generated OpenAPI sources
Compile / unmanagedSourceDirectories += (file(".") / "clients" / "delta-rest" / "target" / "src" / "main" / "java"),
libraryDependencies ++= Seq(
"com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion,
"com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion,
"com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion,
"com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonVersion,
"org.openapitools" % "jackson-databind-nullable" % openApiToolsJacksonBindNullableVersion,
"com.google.code.findbugs" % "jsr305" % "3.0.2",
"jakarta.annotation" % "jakarta.annotation-api" % "3.0.0" % Provided,

// Test dependencies
"org.mockito" % "mockito-core" % "5.11.0" % Test,
"org.mockito" % "mockito-inline" % "5.2.0" % Test,
"org.mockito" % "mockito-junit-jupiter" % "5.12.0" % Test,
"org.junit.jupiter" % "junit-jupiter" % "5.10.3" % Test,
"net.aichler" % "jupiter-interface" % JupiterKeys.jupiterVersion.value % Test,
"org.assertj" % "assertj-core" % "3.26.3" % Test,
),
(Compile / compile) := ((Compile / compile) dependsOn generate).value,

// Add custom test sources from clients/delta-rest directory
Test / unmanagedSourceDirectories += (file(".") / "clients" / "delta-rest" / "src" / "test" / "java"),

// OpenAPI generation specs
openApiInputSpec := (file(".") / "api" / "delta-rest.yaml").toString,
openApiGeneratorName := "java",
openApiOutputDir := (file(".") / "clients" / "delta-rest" / "target").toString,
openApiApiPackage := s"$orgName.client.deltarest.api",
openApiModelPackage := s"$orgName.client.deltarest.model",
openApiAdditionalProperties := Map(
"library" -> "native",
"useJakartaEe" -> "true",
"hideGenerationTimestamp" -> "true",
"openApiNullable" -> "false",
"enumUnknownDefaultCase" -> "true"),
openApiGenerateApiTests := SettingDisabled,
openApiGenerateModelTests := SettingDisabled,
openApiGenerateApiDocumentation := SettingDisabled,
openApiGenerateModelDocumentation := SettingDisabled,
// Define the simple generate command to generate full client codes
generate := {
val _ = openApiGenerate.value

// Delete the generated build.sbt file so that it is not used for our sbt config
val buildSbtFile = file(openApiOutputDir.value) / "build.sbt"
if (buildSbtFile.exists()) {
buildSbtFile.delete()
}
},
)

lazy val cli = (project in file("examples") / "cli")
.dependsOn(server % "test->test")
.dependsOn(serverModels)
.dependsOn(deltaRestServerModels)
.dependsOn(deltaRestClient)
.dependsOn(client % "compile->compile;test->test")
.dependsOn(controlApi % "compile->compile")
.enablePlugins(CheckstylePlugin)
Expand Down Expand Up @@ -562,7 +661,7 @@ lazy val serverShaded = (project in file("server-shaded"))
)

lazy val spark = (project in file("connectors/spark"))
.dependsOn(client)
.dependsOn(client, deltaRestClient)
.enablePlugins(CheckstylePlugin)
.settings(
name := s"$artifactNamePrefix-spark",
Expand Down Expand Up @@ -687,7 +786,7 @@ lazy val integrationTests = (project in file("integration-tests"))
)

lazy val root = (project in file("."))
.aggregate(serverModels, client, pythonClient, server, cli, spark, controlApi, controlModels, apiDocs)
.aggregate(serverModels, client, pythonClient, server, cli, spark, controlApi, controlModels, deltaRestServerModels, deltaRestClient, apiDocs)
.settings(
name := s"$artifactNamePrefix",
createTarballSettings(),
Expand Down
Loading