Debezium + Flink Oracle CDC - «Тема истории DB или ее содержание полностью или частично отсутствует» для некоторых табли ⇐ JAVA

[Anonymous]

Я использую Flink с Debezium, чтобы потреблять изменения CDC из таблиц Oracle DB через logminer. < /p>
Для некоторых таблиц все работает нормально. Например, следующая таблица работает без проблем: < /p>

Код: Выделить всё

CREATE TABLE CDC_PRODUCTS (
ID NUMBER,
NAME VARCHAR2(100),
DESCRIPTION VARCHAR2(200),
WEIGHT NUMBER(10, 3)
);
< /code>
Однако для других таблиц я получаю следующую ошибку: < /p>
io.debezium.DebeziumException: The db history topic or its content is fully or partially missing. Please check database history topic configuration and re-execute the snapshot.
< /code>
Tulck Stack Trace: < /p>
io.debezium.DebeziumException: The db history topic or its content is fully or partially missing. Please check database history topic configuration and re-execute the snapshot.
at io.debezium.relational.HistorizedRelationalDatabaseSchema.recover(HistorizedRelationalDatabaseSchema.java:59)
at org.apache.flink.cdc.connectors.oracle.source.reader.fetch.OracleSourceFetchTaskContext.validateAndLoadDatabaseHistory(OracleSourceFetchTaskContext.java:275)
at org.apache.flink.cdc.connectors.oracle.source.reader.fetch.OracleSourceFetchTaskContext.configure(OracleSourceFetchTaskContext.java:118)
at org.apache.flink.cdc.connectors.base.source.reader.external.IncrementalSourceStreamFetcher.submitTask(IncrementalSourceStreamFetcher.java:84)
at org.apache.flink.cdc.connectors.base.source.reader.IncrementalSourceSplitReader.submitStreamSplit(IncrementalSourceSplitReader.java:261)
at org.apache.flink.cdc.connectors.base.source.reader.IncrementalSourceSplitReader.pollSplitRecords(IncrementalSourceSplitReader.java:153)
at org.apache.flink.cdc.connectors.base.source.reader.IncrementalSourceSplitReader.fetch(IncrementalSourceSplitReader.java:98)
at org.apache.flink.connector.base.source.reader.fetcher.FetchTask.run(FetchTask.java:58)
at org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher.runOnce(SplitFetcher.java:162)
at org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher.run(SplitFetcher.java:114)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)

20:13:49.779 [Source Data Fetcher for Source: OracleParallelSource (1/1)#92] ERROR org.apache.flink.connector.base.source.reader.fetcher.SplitFetcherManager - Received uncaught exception.
java.lang.RuntimeException: SplitFetcher thread 0 received unexpected exception while polling the records
at org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher.runOnce(SplitFetcher.java:165)
at org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher.run(SplitFetcher.java:114)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)

Caused by: java.io.IOException: io.debezium.DebeziumException: The db history topic or its content is fully or partially missing. Please check database history topic configuration and re-execute the snapshot.
at org.apache.flink.cdc.connectors.base.source.reader.IncrementalSourceSplitReader.fetch(IncrementalSourceSplitReader.java:101)
at org.apache.flink.connector.base.source.reader.fetcher.FetchTask.run(FetchTask.java:58)
at org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher.runOnce(SplitFetcher.java:162)
... 7 more

Проблемная таблица имеет:

• 11 столбцов;
• Основное ограничение первичного ключа;
  
  6 не Null code. />

Вопросы:

Можно ли вызвать отсутствующие снимки или метаданные для этой конкретной таблицы? /> < /li>
Любые предложения по восстановлению или повторной повторности истории DB для таких случаев?

Код: Выделить всё

CREATE TABLE "SI_HCMP_REPLICA1"."*****_HISTORY"
(    "****_ID" NUMBER DEFAULT TO_NUMBER(TO_CHAR(SYSDATE@!,'YYYYMMDDHH24MI')||TO_CHAR(****.****.NEXTVAL)),
"CUSTOMER_ID" NUMBER NOT NULL ENABLE,
"ASSET_ID" NUMBER NOT NULL ENABLE,
"ASSET_LAST_MODIFIED_ON" DATE NOT NULL ENABLE,
"CREATED_ON" DATE DEFAULT SYSDATE NOT NULL ENABLE,
"CREATED_BY_ID" VARCHAR2(100),
"UPDATED_ON" DATE,
"UPDATED_BY_ID" VARCHAR2(100),
"PROCESS_STATUS" VARCHAR2(1) DEFAULT 'N' NOT NULL ENABLE,
"ASSET_TECH_ID" NUMBER,
"WORKFLOW_FLAG" VARCHAR2(1) DEFAULT 'N' NOT NULL ENABLE,
CONSTRAINT "RM_ASSET_HISTORY_PK" PRIMARY KEY ("****_ID")
USING INDEX PCTFREE 10 INITRANS 2 MAXTRANS 255 COMPUTE STATISTICS
STORAGE(INITIAL 65536 NEXT 1048576 MINEXTENTS 1 MAXEXTENTS 2147483645
PCTINCREASE 0 FREELISTS 1 FREELIST GROUPS 1
BUFFER_POOL DEFAULT FLASH_CACHE DEFAULT CELL_FLASH_CACHE DEFAULT)
TABLESPACE "HCMP_DATA"  ENABLE,
SUPPLEMENTAL LOG DATA (ALL) COLUMNS,
SUPPLEMENTAL LOG GROUP "*****_HISTORY" ("ASSET_ID", "CUSTOMER_ID", "CREATED_ON") ALWAYS
) PCTFREE 10 PCTUSED 40 INITRANS 1 MAXTRANS 255
STORAGE(
BUFFER_POOL DEFAULT FLASH_CACHE DEFAULT CELL_FLASH_CACHE DEFAULT)
TABLESPACE "HCMP_DATA"
PARTITION BY RANGE ("CREATED_ON") INTERVAL (NUMTODSINTERVAL (1,'DAY'))
SUBPARTITION BY LIST ("PROCESS_STATUS")
SUBPARTITION TEMPLATE (
SUBPARTITION "PROCESSED" VALUES ( 'Y' ),
SUBPARTITION "UNPROCESSED" VALUES ( 'N' ) )
(PARTITION "SYS_P164554"  VALUES LESS THAN (TO_DATE(' 2023-12-22 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN'))
PCTFREE 10 PCTUSED 40 INITRANS 1 MAXTRANS 255
STORAGE(
BUFFER_POOL DEFAULT FLASH_CACHE DEFAULT CELL_FLASH_CACHE DEFAULT)
TABLESPACE "HCMP_DATA"
( SUBPARTITION "SYS_SUBP164553"  VALUES ('N') SEGMENT CREATION IMMEDIATE
STORAGE(
BUFFER_POOL DEFAULT FLASH_CACHE DEFAULT CELL_FLASH_CACHE DEFAULT)
TABLESPACE "HCMP_DATA"
NOCOMPRESS ,
SUBPARTITION "PROCESSED_ASSET_SYS_P164554"  VALUES ('Y') SEGMENT CREATION DEFERRED
STORAGE(
BUFFER_POOL DEFAULT FLASH_CACHE DEFAULT CELL_FLASH_CACHE DEFAULT)
TABLESPACE "****_DATA"
NOCOMPRESS ) )  ENABLE ROW MOVEMENT

после дальнейшего исследования это могло быть связано с разделениями на таблице.

Подробнее здесь: https://stackoverflow.com/questions/797 ... y-or-parti