Skip to content

feat: add crawling provider #52

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nirolevy wants to merge 3 commits into
base: master
Choose a base branch
from
Open

feat: add crawling provider #52

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5b0f0d1
feat(providers): add function to get file (copied from file-syncer)
nirolevy Sep 16, 2025
562de30
feat(providers): add JSON crawling provider
nirolevy Sep 16, 2025
b11b250
feat(providers): add crawling provider loading
nirolevy Sep 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions config/default.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@
"NFS": {
"pvPath": "/Path/To/Models"
},
"crawling": {
"extension": ".json",
"nestedJsonPath": "$.root..uri",
Copy link
Copy Markdown
Contributor

@asafMasa asafMasa Oct 22, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please add support for both "uri" and "url" (old legacy 3Dtiles models)

"ignoreNotFound": true,
"underlying": "NFS"
},
"ingestion": {
"provider": "NFS",
"blackList": ["tar", "zip", "rar", "7z"]
Expand Down
6 changes: 6 additions & 0 deletions config/test.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
{
"crawling": {
"extension": ".json",
"nestedJsonPath": "$.root..uri",
"ignoreNotFound": true,
"underlying": "NFS"
},
"S3": {
"accessKeyId": "minioadmin",
"secretAccessKey": "minioadmin",
Expand Down
145 changes: 139 additions & 6 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,12 @@
"@map-colonies/error-express-handler": "^2.1.0",
"@map-colonies/express-access-log-middleware": "^2.0.1",
"@map-colonies/js-logger": "^1.0.1",
"@map-colonies/types": "^1.7.0",
"@map-colonies/mc-model-types": "^19.0.0",
"@map-colonies/mc-priority-queue": "^8.2.1",
"@map-colonies/openapi-express-viewer": "^3.0.0",
"@map-colonies/read-pkg": "0.0.1",
"@map-colonies/telemetry": "^6.1.0",
"@map-colonies/types": "^1.7.0",
"@opentelemetry/api": "1.7.0",
"@opentelemetry/api-metrics": "0.23.0",
"@opentelemetry/context-async-hooks": "^1.24.1",
Expand All @@ -65,6 +65,7 @@
"express": "^4.18.2",
"express-openapi-validator": "^5.0.4",
"http-status-codes": "^2.2.0",
"jsonpath": "^1.1.1",
"n-readlines": "^1.0.1",
"prom-client": "^15.1.1",
"reflect-metadata": "^0.1.13",
Expand All @@ -76,12 +77,13 @@
"@faker-js/faker": "^8.4.1",
"@map-colonies/eslint-config": "^4.0.0",
"@map-colonies/prettier-config": "0.0.1",
"@redocly/openapi-cli": "^1.0.0-beta.94",
"@redocly/cli": "^1.34.3",
"@redocly/openapi-cli": "^1.0.0-beta.94",
"@types/compression": "^1.7.2",
"@types/config": "^3.3.0",
"@types/express": "^4.17.17",
"@types/jest": "^29.5.2",
"@types/jsonpath": "^0.2.4",
"@types/multer": "^1.4.7",
"@types/n-readlines": "^1.0.3",
"@types/supertest": "^2.0.12",
Expand Down
1 change: 1 addition & 0 deletions src/common/constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,6 @@ export const SERVICES: Record<string, symbol> = {
PROVIDER_CONFIG: Symbol('ProviderConfig'),
QUEUE_FILE_HANDLER: Symbol('QueueFileHandler'),
JOB_MANAGER_CLIENT: Symbol('JobManagerClient'),
UNDERLYING: Symbol('Underlying'),
};
/* eslint-enable @typescript-eslint/naming-convention */
11 changes: 10 additions & 1 deletion src/common/interfaces.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@ export interface DeletePayload {
producerName: string;
}

// ToDo: merge this class with the identical class in file-syncer
export interface Provider {
streamModelPathsToQueueFile: (modelId: string, pathToTileset: string, productName: string) => Promise<number>;
getFile: (filePath: string) => Promise<Buffer>;
}

export interface IngestionJobParameters {
Expand Down Expand Up @@ -70,7 +72,14 @@ export interface NFSConfig {
pvPath: string;
}

export type ProviderConfig = S3Config | NFSConfig;
export interface CrawlingConfig {
extension: string;
nestedJsonPath: string;
ignoreNotFound?: boolean;
underlying?: string;
}

export type ProviderConfig = S3Config | NFSConfig | CrawlingConfig;

export interface JobOperationResponse {
jobId: string;
Expand Down
9 changes: 2 additions & 7 deletions src/containerConfig.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import jsLogger, { LoggerOptions } from '@map-colonies/js-logger';
import client from 'prom-client';
import { JobManagerClient } from '@map-colonies/mc-priority-queue';
import { SERVICES, SERVICE_NAME } from './common/constants';
import { Provider, ProviderConfig } from './common/interfaces';
import { tracing } from './common/tracing';
import { jobOperationsRouterFactory, JOB_OPERATIONS_ROUTER_SYMBOL } from './jobOperations/routes/jobOperationsRouter';
import { InjectionObject, registerDependencies } from './common/dependencyRegistration';
Expand Down Expand Up @@ -62,18 +61,14 @@ export const registerExternalValues = (options?: RegisterOptions): DependencyCon
{
token: SERVICES.PROVIDER_CONFIG,
provider: {
useFactory: (): ProviderConfig => {
return getProviderConfig(provider);
},
useFactory: (container) => getProviderConfig(container),
},
},
{ token: SERVICES.QUEUE_FILE_HANDLER, provider: { useClass: QueueFileHandler } },
{
token: SERVICES.PROVIDER,
provider: {
useFactory: (): Provider => {
return getProvider(provider);
},
useFactory: (container) => getProvider(provider, container),
},
},
{
Expand Down
Loading
Loading