diff --git a/bindings/c/src/catalog.rs b/bindings/c/src/catalog.rs index ba2fdeba3..b94a0dfed 100644 --- a/bindings/c/src/catalog.rs +++ b/bindings/c/src/catalog.rs @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::ffi::{c_char, c_void}; use std::sync::Arc; use paimon::catalog::Identifier; +use paimon::spec::Schema; use paimon::{Catalog, CatalogFactory, Options}; use crate::error::{check_non_null, paimon_error, validate_cstr, PaimonErrorCode}; @@ -101,6 +103,97 @@ pub unsafe extern "C" fn paimon_catalog_free(catalog: *mut paimon_catalog) { } } +/// Create a database. Returns null on success, or an error to free with +/// `paimon_error_free`. A null `properties_json` or JSON null means no properties. +/// +/// # Safety +/// `catalog` must be a valid catalog handle from a previous paimon C call. +/// `name` and any non-null `properties_json` must be valid null-terminated C strings. +#[no_mangle] +pub unsafe extern "C" fn paimon_catalog_create_database( + catalog: *const paimon_catalog, + name: *const c_char, + properties_json: *const c_char, + ignore_if_exists: bool, +) -> *mut paimon_error { + if let Err(error) = check_non_null(catalog, "catalog") { + return error; + } + let name = match validate_cstr(name, "name") { + Ok(name) => name, + Err(error) => return error, + }; + let properties = if properties_json.is_null() { + HashMap::new() + } else { + let json = match validate_cstr(properties_json, "properties_json") { + Ok(json) => json, + Err(error) => return error, + }; + match serde_json::from_str::>>(&json) { + Ok(properties) => properties.unwrap_or_default(), + Err(error) => { + return paimon_error::new( + PaimonErrorCode::InvalidInput, + format!("Failed to parse database properties JSON: {error}"), + ); + } + } + }; + + let catalog = &*((*catalog).inner as *const Arc); + match runtime().block_on(catalog.create_database(&name, ignore_if_exists, properties)) { + Ok(()) => std::ptr::null_mut(), + Err(error) => paimon_error::from_paimon(error), + } +} + +/// Create a table from Paimon's DDL Schema JSON. Returns null on success, or +/// an error to free with `paimon_error_free`. +/// +/// # Safety +/// `catalog` and `identifier` must be valid handles from previous paimon C calls. +/// `schema_json` must be a valid null-terminated C string. +#[no_mangle] +pub unsafe extern "C" fn paimon_catalog_create_table( + catalog: *const paimon_catalog, + identifier: *const paimon_identifier, + schema_json: *const c_char, + ignore_if_exists: bool, +) -> *mut paimon_error { + if let Err(error) = check_non_null(catalog, "catalog") { + return error; + } + if let Err(error) = check_non_null(identifier, "identifier") { + return error; + } + let json = match validate_cstr(schema_json, "schema_json") { + Ok(json) => json, + Err(error) => return error, + }; + let schema = match serde_json::from_str::(&json) { + Ok(schema) => schema, + Err(error) => { + return paimon_error::new( + PaimonErrorCode::InvalidInput, + format!("Failed to parse table schema JSON: {error}"), + ); + } + }; + // Deserialization does not run Schema's normalization and validation. + let schema = match schema.copy(schema.row_type()) { + Ok(schema) => schema, + Err(error) => return paimon_error::from_paimon(error), + }; + + let catalog = &*((*catalog).inner as *const Arc); + let identifier = &*((*identifier).inner as *const Identifier); + match runtime().block_on(catalog.create_table(identifier, schema, ignore_if_exists)) { + Ok(()) => std::ptr::null_mut(), + Err(error) => paimon_error::from_paimon(error), + } +} + /// Get a table from the catalog. /// /// # Safety diff --git a/bindings/go/catalog.go b/bindings/go/catalog.go index cadde62c0..1d82ec275 100644 --- a/bindings/go/catalog.go +++ b/bindings/go/catalog.go @@ -21,6 +21,7 @@ package paimon import ( "context" + "encoding/json" "runtime" "sync" "unsafe" @@ -68,6 +69,30 @@ func (c *Catalog) Close() { }) } +// CreateDatabase creates a database with optional properties. When ignoreIfExists +// is true, an existing database is accepted. +func (c *Catalog) CreateDatabase(name string, properties map[string]string, ignoreIfExists bool) error { + if c.inner == nil { + return ErrClosed + } + return ffiCatalogCreateDatabase.symbol(c.ctx)(c.inner, name, properties, ignoreIfExists) +} + +// CreateTable creates a table from the Paimon schema JSON used by the REST +// catalog's CreateTableRequest.schema. The database must already exist. +// When ignoreIfExists is true, an existing table is accepted. +func (c *Catalog) CreateTable(id Identifier, schemaJSON []byte, ignoreIfExists bool) error { + if c.inner == nil { + return ErrClosed + } + cID, err := c.newIdentifier(id) + if err != nil { + return err + } + defer ffiIdentifierFree.symbol(c.ctx)(cID) + return ffiCatalogCreateTable.symbol(c.ctx)(c.inner, cID, schemaJSON, ignoreIfExists) +} + // GetTable retrieves a table from the catalog using the given identifier. func (c *Catalog) GetTable(id Identifier) (*Table, error) { if c.inner == nil { @@ -204,6 +229,69 @@ var ffiIdentifierFree = newFFI(ffiOpts{ } }) +var ffiCatalogCreateDatabase = newFFI(ffiOpts{ + sym: "paimon_catalog_create_database", + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer, &ffi.TypeUint8}, +}, func(ctx context.Context, ffiCall ffiCall) func(*paimonCatalog, string, map[string]string, bool) error { + return func(catalog *paimonCatalog, name string, properties map[string]string, ignoreIfExists bool) error { + namePtr, err := bytePtrFromString(name) + if err != nil { + return err + } + propertiesJSON, err := json.Marshal(properties) + if err != nil { + return err + } + propertiesPtr, err := bytePtrFromString(string(propertiesJSON)) + if err != nil { + return err + } + ignore := uint8(0) + if ignoreIfExists { + ignore = 1 + } + var ffiError *paimonError + ffiCall( + unsafe.Pointer(&ffiError), + unsafe.Pointer(&catalog), + unsafe.Pointer(&namePtr), + unsafe.Pointer(&propertiesPtr), + unsafe.Pointer(&ignore), + ) + runtime.KeepAlive(namePtr) + runtime.KeepAlive(propertiesPtr) + return parseError(ctx, ffiError) + } +}) + +var ffiCatalogCreateTable = newFFI(ffiOpts{ + sym: "paimon_catalog_create_table", + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer, &ffi.TypeUint8}, +}, func(ctx context.Context, ffiCall ffiCall) func(*paimonCatalog, *paimonIdentifier, []byte, bool) error { + return func(catalog *paimonCatalog, id *paimonIdentifier, schemaJSON []byte, ignoreIfExists bool) error { + schemaPtr, err := bytePtrFromString(string(schemaJSON)) + if err != nil { + return err + } + ignore := uint8(0) + if ignoreIfExists { + ignore = 1 + } + var ffiError *paimonError + ffiCall( + unsafe.Pointer(&ffiError), + unsafe.Pointer(&catalog), + unsafe.Pointer(&id), + unsafe.Pointer(&schemaPtr), + unsafe.Pointer(&ignore), + ) + runtime.KeepAlive(schemaPtr) + return parseError(ctx, ffiError) + } +}) + var ffiCatalogGetTable = newFFI(ffiOpts{ sym: "paimon_catalog_get_table", rType: &typeResultGetTable, diff --git a/bindings/go/tests/catalog_test.go b/bindings/go/tests/catalog_test.go new file mode 100644 index 000000000..8819792d3 --- /dev/null +++ b/bindings/go/tests/catalog_test.go @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package paimon_test + +import ( + "errors" + "slices" + "testing" + + paimon "github.com/apache/paimon-rust/bindings/go" +) + +func TestCatalogCreateDatabaseAndTable(t *testing.T) { + catalog, err := paimon.NewCatalog(map[string]string{"warehouse": t.TempDir()}) + if err != nil { + t.Fatal(err) + } + defer catalog.Close() + + requireCode := func(err error, code paimon.ErrorCode) { + t.Helper() + var paimonErr *paimon.Error + if !errors.As(err, &paimonErr) || paimonErr.Code() != code { + t.Fatalf("got error %v, want Paimon error code %d", err, code) + } + } + schema := []byte(`{ + "fields": [ + {"id": 0, "name": "id", "type": "INT"}, + {"id": 1, "name": "name", "type": "STRING"} + ], + "partitionKeys": [], + "primaryKeys": ["id"], + "options": {"bucket": "1", "merge-engine": "partial-update"} + }`) + id := paimon.NewIdentifier("demo", "users") + requireCode(catalog.CreateTable(id, schema, false), paimon.CodeNotFound) + requireCode(catalog.CreateDatabase("demo", map[string]string{"location": "/unused"}, false), paimon.CodeInvalidInput) + if err := catalog.CreateDatabase("demo", nil, false); err != nil { + t.Fatal(err) + } + requireCode(catalog.CreateDatabase("demo", nil, false), paimon.CodeAlreadyExist) + if err := catalog.CreateDatabase("demo", nil, true); err != nil { + t.Fatal(err) + } + databases, err := catalog.ListDatabases() + if err != nil || !slices.Contains(databases, "demo") { + t.Fatalf("ListDatabases = %v, %v; want demo", databases, err) + } + + requireCode(catalog.CreateTable(id, []byte("{"), false), paimon.CodeInvalidInput) + invalidSchema := []byte(`{ + "fields": [{"id": 0, "name": "id", "type": "INT"}], + "partitionKeys": [], + "primaryKeys": ["missing"], + "options": {"bucket": "1"} + }`) + requireCode(catalog.CreateTable(id, invalidSchema, false), paimon.CodeInvalidInput) + requireCode(getTableError(catalog, id), paimon.CodeNotFound) + + if err := catalog.CreateTable(id, schema, false); err != nil { + t.Fatal(err) + } + requireCode(catalog.CreateTable(id, schema, false), paimon.CodeAlreadyExist) + if err := catalog.CreateTable(id, schema, true); err != nil { + t.Fatal(err) + } + + table, err := catalog.GetTable(id) + if err != nil { + t.Fatal(err) + } + defer table.Close() + + builder, err := table.NewWriteBuilder() + if err != nil { + t.Fatal(err) + } + defer builder.Close() + writer, err := builder.NewWrite() + if err != nil { + t.Fatal(err) + } + defer writer.Close() + + want := []row{{id: 1, name: "Ada"}} + record := makeRecord(t, want) + if err := writer.WriteArrowBatch(record); err != nil { + record.Release() + t.Fatal(err) + } + record.Release() + messages, err := writer.PrepareCommit() + if err != nil { + t.Fatal(err) + } + defer messages.Close() + commit, err := builder.NewCommit() + if err != nil { + t.Fatal(err) + } + defer commit.Close() + if err := commit.Commit(messages); err != nil { + t.Fatal(err) + } + if got := readTableRows(t, table); !slices.Equal(got, want) { + t.Fatalf("rows = %v, want %v", got, want) + } + + catalog.Close() + if err := catalog.CreateDatabase("demo", nil, false); !errors.Is(err, paimon.ErrClosed) { + t.Fatalf("CreateDatabase on closed catalog = %v, want ErrClosed", err) + } + if err := catalog.CreateTable(id, schema, false); !errors.Is(err, paimon.ErrClosed) { + t.Fatalf("CreateTable on closed catalog = %v, want ErrClosed", err) + } +} + +func getTableError(catalog *paimon.Catalog, id paimon.Identifier) error { + table, err := catalog.GetTable(id) + if table != nil { + table.Close() + } + return err +} diff --git a/docs/src/go-binding.md b/docs/src/go-binding.md index de780ed62..698ac66bf 100644 --- a/docs/src/go-binding.md +++ b/docs/src/go-binding.md @@ -297,6 +297,34 @@ catalog, err := paimon.NewCatalog(map[string]string{ }) ``` +## Creating a Database and Table + +Use `CreateDatabase` and `CreateTable` to provision tables from Go. Pass the +schema JSON used by the Paimon REST create-table request's `schema` field. +The last argument controls whether an existing database or table is accepted. + +```go +if err := catalog.CreateDatabase("demo", nil, true); err != nil { + log.Fatal(err) +} + +schemaJSON := []byte(`{ + "fields": [ + {"id": 0, "name": "id", "type": "INT NOT NULL"}, + {"id": 1, "name": "name", "type": "STRING"} + ], + "partitionKeys": [], + "primaryKeys": ["id"], + "options": {"bucket": "1", "merge-engine": "partial-update"} +}`) +id := paimon.NewIdentifier("demo", "users") +if err := catalog.CreateTable(id, schemaJSON, true); err != nil { + log.Fatal(err) +} +``` + +Open the new table with `catalog.GetTable(id)`. + ## Managing Tags Catalog tags pin table snapshots. Pass `nil` to tag the latest snapshot, then