-
Notifications
You must be signed in to change notification settings - Fork 37
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add Control file module files and validation #2445
base: master
Are you sure you want to change the base?
Changes from 43 commits
753618b
8d39d02
bf94c49
47be388
913eb1c
1f204b8
6cfa83a
d381b2b
67f2474
14e3593
a096d51
dbf1940
cd8add9
52890c8
5114639
4f9cd75
1997eb8
91e6310
8a7338b
2b52eeb
e206073
26d3144
b86487d
818a2b4
90c4105
3d5d3e0
6495202
90abd9e
ba2b3dd
b1b811b
30db988
03324e1
ccb1ace
a374f1a
a65c9b5
acedabe
bf31a01
57cd330
7a39564
093cb1d
bfebd95
fd1c186
cebb543
8ecb39c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -719,6 +719,50 @@ public enum CoreError implements ScalarDbError { | |||||
"", | ||||||
""), | ||||||
|
||||||
DATA_LOADER_DUPLICATE_DATA_MAPPINGS( | ||||||
Category.USER_ERROR, | ||||||
"0158", | ||||||
"Duplicate data mappings found for table '%s' in the control file", | ||||||
"", | ||||||
""), | ||||||
DATA_LOADER_MISSING_COLUMN_MAPPING( | ||||||
Category.USER_ERROR, | ||||||
"0159", | ||||||
"No mapping found for column '%s' in table '%s' in the control file. \\nControl file validation set at 'FULL'. All columns need to be mapped.", | ||||||
"", | ||||||
""), | ||||||
DATA_LOADER_CONTROL_FILE_MISSING_DATA_MAPPINGS( | ||||||
Category.USER_ERROR, "0160", "The control file is missing data mappings", "", ""), | ||||||
DATA_LOADER__MISSING_NAMESPACE_OR_TABLE( | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
Category.USER_ERROR, | ||||||
"0161", | ||||||
"The provided namespace '%s' and/or table name '%s' is incorrect and could not be found", | ||||||
"", | ||||||
""), | ||||||
DATA_LOADER_TARGET_COLUMN_NOT_FOUND( | ||||||
Category.USER_ERROR, | ||||||
"0162", | ||||||
"The target column '%s' for source field '%s' could not be found in table '%s'", | ||||||
"", | ||||||
""), | ||||||
DATA_LOADER_MISSING_PARTITION_KEY( | ||||||
Category.USER_ERROR, | ||||||
"0163", | ||||||
"The required partition key '%s' is missing in the control file mapping for table '%s'", | ||||||
"", | ||||||
""), | ||||||
DATA_LOADER_MISSING_CLUSTERING_KEY( | ||||||
Category.USER_ERROR, | ||||||
"0164", | ||||||
"The required clustering key '%s' is missing in the control file mapping for table '%s'", | ||||||
"", | ||||||
""), | ||||||
DATA_LOADER_MULTIPLE_MAPPINGS_FOR_COLUMN_FOUND( | ||||||
Category.USER_ERROR, | ||||||
"0165", | ||||||
"Multiple data mappings found for column '%s' in table '%s'", | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
"Duplicated" sounds a bit clear? |
||||||
"", | ||||||
""), | ||||||
// | ||||||
// Errors for the concurrency error category | ||||||
// | ||||||
|
@@ -969,6 +1013,18 @@ public enum CoreError implements ScalarDbError { | |||||
"Handling the before-preparation snapshot hook failed. Details: %s", | ||||||
"", | ||||||
""), | ||||||
DATA_LOADER_ERROR_CRUD_EXCEPTION( | ||||||
Category.INTERNAL_ERROR, | ||||||
"0047", | ||||||
"Something went wrong while trying to save the data. Details %s", | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
"", | ||||||
""), | ||||||
DATA_LOADER_ERROR_SCAN( | ||||||
Category.INTERNAL_ERROR, | ||||||
"0048", | ||||||
"Something went wrong while scanning. Are you sure you are running in the correct transaction mode? Details %s", | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
"", | ||||||
""), | ||||||
|
||||||
// | ||||||
// Errors for the unknown transaction status error category | ||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
package com.scalar.db.dataloader.core.dataimport.controlfile; | ||
|
||
import com.fasterxml.jackson.annotation.JsonCreator; | ||
import com.fasterxml.jackson.annotation.JsonProperty; | ||
import java.util.ArrayList; | ||
import java.util.List; | ||
import lombok.Getter; | ||
import lombok.Setter; | ||
|
||
/** Represents the control file */ | ||
@Getter | ||
@Setter | ||
public class ControlFile { | ||
|
||
@JsonProperty("tables") | ||
private final List<ControlFileTable> tables; | ||
|
||
/** Class constructor */ | ||
public ControlFile() { | ||
this.tables = new ArrayList<>(); | ||
} | ||
|
||
@JsonCreator | ||
public ControlFile(@JsonProperty("tables") List<ControlFileTable> tables) { | ||
this.tables = tables; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
package com.scalar.db.dataloader.core.dataimport.controlfile; | ||
|
||
import com.fasterxml.jackson.annotation.JsonCreator; | ||
import com.fasterxml.jackson.annotation.JsonProperty; | ||
import java.util.ArrayList; | ||
import java.util.List; | ||
import lombok.Getter; | ||
import lombok.Setter; | ||
|
||
/** | ||
* Represents the configuration for a single table in the control file, including its namespace, | ||
* table name, and field mappings. This class is used to define how data from a control file maps to | ||
* a specific table in ScalarDB. | ||
*/ | ||
@Getter | ||
@Setter | ||
public class ControlFileTable { | ||
|
||
/** The namespace of the table in ScalarDB. */ | ||
@JsonProperty("namespace") | ||
private String namespace; | ||
|
||
/** The name of the table in ScalarDB. */ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
@JsonProperty("table_name") | ||
private String tableName; | ||
|
||
/** | ||
* A list of mappings defining the correspondence between control file fields and table columns. | ||
*/ | ||
@JsonProperty("mappings") | ||
private final List<ControlFileTableFieldMapping> mappings; | ||
|
||
/** | ||
* Creates a new {@code ControlFileTable} instance with the specified namespace and table name. | ||
* The mappings list is initialized as an empty list. | ||
* | ||
* @param namespace The namespace of the table in ScalarDB. | ||
* @param tableName The name of the table in ScalarDB. | ||
*/ | ||
public ControlFileTable(String namespace, String tableName) { | ||
this.namespace = namespace; | ||
this.tableName = tableName; | ||
this.mappings = new ArrayList<>(); | ||
} | ||
|
||
/** | ||
* Constructs a {@code ControlFileTable} instance using data from a serialized JSON object. This | ||
* constructor is used for deserialization of API requests or control files. | ||
* | ||
* @param namespace The namespace of the table in ScalarDB. | ||
* @param tableName The name of the table in ScalarDB. | ||
* @param mappings A list of mappings that define the relationship between control file fields and | ||
* table columns. | ||
*/ | ||
@JsonCreator | ||
public ControlFileTable( | ||
@JsonProperty("namespace") String namespace, | ||
@JsonProperty("table_name") String tableName, | ||
@JsonProperty("mappings") List<ControlFileTableFieldMapping> mappings) { | ||
this.namespace = namespace; | ||
this.tableName = tableName; | ||
this.mappings = mappings; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
package com.scalar.db.dataloader.core.dataimport.controlfile; | ||
|
||
import com.fasterxml.jackson.annotation.JsonCreator; | ||
import com.fasterxml.jackson.annotation.JsonProperty; | ||
import lombok.Getter; | ||
import lombok.Setter; | ||
|
||
/** | ||
* Represents the mapping of a single field in the control file to a column in a ScalarDB table. | ||
* This class defines how data from a specific field in the input source should be mapped to the | ||
* corresponding column in the database. | ||
*/ | ||
@Getter | ||
@Setter | ||
public class ControlFileTableFieldMapping { | ||
|
||
/** The name of the field in the input source (e.g., JSON or CSV). */ | ||
@JsonProperty("source_field") | ||
private String sourceField; | ||
|
||
/** The name of the column in the ScalarDB table that the field maps to. */ | ||
@JsonProperty("target_column") | ||
private String targetColumn; | ||
|
||
/** | ||
* Constructs a {@code ControlFileTableFieldMapping} instance using data from a serialized JSON | ||
* object. This constructor is primarily used for deserialization of control file mappings. | ||
* | ||
* @param sourceField The name of the field in the input source (e.g., JSON or CSV). | ||
* @param targetColumn The name of the corresponding column in the ScalarDB table. | ||
*/ | ||
@JsonCreator | ||
public ControlFileTableFieldMapping( | ||
@JsonProperty("source_field") String sourceField, | ||
@JsonProperty("target_column") String targetColumn) { | ||
this.sourceField = sourceField; | ||
this.targetColumn = targetColumn; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
package com.scalar.db.dataloader.core.dataimport.controlfile; | ||
|
||
/** Represents the control file */ | ||
public class ControlFileValidationException extends Exception { | ||
|
||
/** | ||
* Class constructor | ||
* | ||
* @param message error message | ||
*/ | ||
public ControlFileValidationException(String message) { | ||
super(message); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ignore this comment if the newline is really necessary.