Skip to content

Commit

Permalink
Added a test to detect COPY ON ERROR clause in the migration assessme…
Browse files Browse the repository at this point in the history 
…nt test (#2157)
  • Loading branch information
@ShivanshGahlot
ShivanshGahlot authored Jan 7, 2025
1 parent 23db698 commit 2cccc27
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 19 deletions.
37 changes: 29 additions & 8 deletions migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,15 @@
},
{
"ObjectType": "SEQUENCE",
"TotalCount": 43,
"TotalCount": 42,
"InvalidCount": 0,
"ObjectNames": "public.employeesforview_id_seq, schema2.employeesforview_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq"
"ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees_employee_id_seq, public.employeesforview_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.employeesforview_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq"
},
{
"ObjectType": "TABLE",
"TotalCount": 82,
"TotalCount": 83,
"InvalidCount": 36,
"ObjectNames": "public.employeesforview, schema2.employeesforview, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2"
"ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employeescopyfromwhere, public.employeescopyonerror, public.employeesforview, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.employeesforview, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2"
},
{
"ObjectType": "INDEX",
Expand Down Expand Up @@ -172,7 +172,8 @@
"public.library_nested",
"public.orders_lateral",
"public.employees",
"public.employees3",
"public.employeescopyfromwhere",
"public.employeescopyonerror",
"public.bigint_multirange_table",
"public.date_multirange_table",
"public.int_multirange_table",
Expand All @@ -187,7 +188,7 @@
"schema2.timestamptz_multirange_table",
"public.employeesforview"
],
"ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 88 objects (80 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.",
"ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 89 objects (81 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.",
"ShardedTables": [
"public.combined_tbl",
"public.citext_type",
Expand Down Expand Up @@ -2111,7 +2112,7 @@
},
{
"SchemaName": "public",
"ObjectName": "employees3",
"ObjectName": "employeescopyfromwhere",
"RowCount": 2,
"ColumnCount": 3,
"Reads": 0,
Expand Down Expand Up @@ -2304,6 +2305,20 @@
"ObjectType": "",
"ParentTableName": null,
"SizeInBytes": 0
},
{
"SchemaName": "public",
"ObjectName": "employeescopyonerror",
"RowCount": 3,
"ColumnCount": 3,
"Reads": 0,
"Writes": 3,
"ReadsPerSecond": 0,
"WritesPerSecond": 0,
"IsIndex": false,
"ObjectType": "",
"ParentTableName": null,
"SizeInBytes": 8192
}

],
Expand Down Expand Up @@ -2507,7 +2522,13 @@
},
{
"ConstructTypeName": "COPY FROM ... WHERE",
"Query": "COPY employees3 (id, name, age)\nFROM STDIN WITH (FORMAT csv)\nWHERE age \u003e 30",
"Query": "COPY employeesCopyFromWhere (id, name, age)\nFROM STDIN WITH (FORMAT csv)\nWHERE age \u003e 30",
"DocsLink": "",
"MinimumVersionsFixedIn": null
},
{
"ConstructTypeName": "COPY ... ON_ERROR",
"Query": "COPY employeesCopyOnError (id, name, age)\nFROM STDIN WITH (FORMAT csv, ON_ERROR IGNORE )",
"DocsLink": "",
"MinimumVersionsFixedIn": null
}
Expand Down
27 changes: 16 additions & 11 deletions migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,28 +142,33 @@ SELECT lo_create('32142');

-- Unsupported COPY constructs

CREATE TABLE IF NOT EXISTS employees3 (
id SERIAL PRIMARY KEY,
CREATE TABLE IF NOT EXISTS employeesCopyFromWhere (
id INT PRIMARY KEY,
name TEXT NOT NULL,
age INT NOT NULL
);


-- COPY FROM with WHERE clause
COPY employees3 (id, name, age)
COPY employeesCopyFromWhere (id, name, age)
FROM STDIN WITH (FORMAT csv)
WHERE age > 30;
1,John Smith,25
2,Jane Doe,34
3,Bob Johnson,31
\.

-- This can be uncommented when we start using PG 17 or later in the tests
-- -- COPY with ON_ERROR clause
-- COPY employees (id, name, age)
-- FROM STDIN WITH (FORMAT csv, ON_ERROR IGNORE );
-- 4,Adam Smith,22
-- 5,John Doe,34
-- 6,Ron Johnson,31
-- \.
CREATE TABLE IF NOT EXISTS employeesCopyOnError (
id INT PRIMARY KEY,
name TEXT NOT NULL,
age INT NOT NULL
);

-- COPY with ON_ERROR clause
COPY employeesCopyOnError (id, name, age)
FROM STDIN WITH (FORMAT csv, ON_ERROR IGNORE );
4,Adam Smith,22
5,John Doe,34
6,Ron Johnson,31
\.

0 comments on commit 2cccc27

Please sign in to comment.