fix: Replicate data from external source to internal source with a Plugin

vitest.config.ts

@@ -1,20 +1,86 @@
-import { defineConfig } from 'vitest/config'
-
-export default defineConfig({
-    assetsInclude: ['**/*.sql'],
-    test: {
-        coverage: {
-            provider: 'istanbul',
-            reporter: ['text', 'html', 'json', 'json-summary', 'lcov'],
-            include: ['src/**/*.ts'],
-            exclude: ['**/node_modules/**'],
-            reportOnFailure: true, // Ensures the report is generated even if tests fail
-            thresholds: {
-                lines: 75,
-                branches: 75,
-                functions: 75,
-                statements: 75,
-            },
-        },
-    },
-})
++++ b/plugins/replicate/index.ts
+@@ -0,0 +1,114 @@
++import { StarbaseApp, StarbaseDBConfiguration } from '../../src/handler';
++import { StarbasePlugin } from '../../src/plugin';
++import { DataSource, QueryResult } from '../../src/types';
++import { createResponse } from '../../src/utils';
+
++interface ReplicationConfig {
++  interval: number;
++  tables: string[];
++  primaryKey: string;
++}
+
++export class ReplicationPlugin extends StarbasePlugin {
++  private config: ReplicationConfig;
++  private dataSource: DataSource;
++  private externalDataSource: DataSource;
+
++  constructor(config: ReplicationConfig) {
++    super('starbasedb:replication');
++    this.config = config;
++  }
+
++  override async register(app: StarbaseApp) {
++    this.dataSource = app.dataSource;
++    this.externalDataSource = await this.getExternalDataSource();
++    this.scheduleReplication();
++  }
+
++  private async getExternalDataSource(): Promise<DataSource> {
++    // Implement logic to get the external data source based on the configuration
++    // For example, using the Outerbase API Key or connection details of the database
++  }
+
++  private async scheduleReplication() {
++    setInterval(async () => {
++      await this.replicateData();
++    }, this.config.interval);
++  }
+
++  private async replicateData() {
++    for (const table of this.config.tables) {
++      const lastSyncedId = await this.getLastSyncedId(table);
++      const newRecords = await this.getExternalRecords(table, lastSyncedId);
++      await this.insertRecords(newRecords);
++    }
++  }
+
++  private async getLastSyncedId(table: string): Promise<number> {
++    const result = await this.dataSource.query(`SELECT MAX(${this.config.primaryKey}) FROM ${table}`);
++    return result.rows[0][this.config.primaryKey];
++  }
+
++  private async getExternalRecords(table: string, lastSyncedId: number): Promise<any[]> {
++    const result = await this.externalDataSource.query(`SELECT * FROM ${table} WHERE ${this.config.primaryKey} > ${lastSyncedId}`);
++    return result.rows;
++  }
+
++  private async insertRecords(records: any[]) {
++    for (const record of records) {
++      await this.dataSource.query(`INSERT INTO ${this.config.tables[0]} (${Object.keys(record).join(', ')}) VALUES (${Object.values(record).map(() => '?').join(', ')})`, Object.values(record));
++    }
++  }
++}
+
++++ b/src/handler.ts
+@@ -10,6 +10,7 @@
+ import { QueryLogPlugin } from '../plugins/query-log/index';
+ import { CronPlugin } from '../plugins/cron/index';
+ import { StatsPlugin } from '../plugins/stats/index';
++import { ReplicationPlugin } from '../plugins/replicate/index';
+
+ export class StarbaseApp {
+   // ...
+ }
+
++++ b/wrangler.toml
+@@ -10,6 +10,10 @@
+ [env]
+ ADMIN_AUTHORIZATION_TOKEN = 'ABC123'
+ CLIENT_AUTHORIZATION_TOKEN = 'DEF456'
++EXTERNAL_DATA_SOURCE = 'postgres://user:password@host:port/database'
++REPLICATION_INTERVAL = 30000
++REPLICATION_TABLES = ['table1', 'table2']
++REPLICATION_PRIMARY_KEY = 'id'
+