From b0804b180a7da857e2bc4c474fa442837300679e Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Thu, 27 May 2021 16:08:17 +0100 Subject: [PATCH 1/6] set precedence to DATABASE_URL --- src/utils/db/hub.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils/db/hub.ts b/src/utils/db/hub.ts index 9e7b50f4..bc684272 100644 --- a/src/utils/db/hub.ts +++ b/src/utils/db/hub.ts @@ -135,7 +135,8 @@ export async function createHub( timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null ) - const postgres = createPostgresPool(serverConfig) + const configOrDatabaseUrl = serverConfig.DATABASE_URL ? serverConfig.DATABASE_URL : serverConfig + const postgres = createPostgresPool(configOrDatabaseUrl) const redisPool = createPool( { From de409c9179d4c975afd2a7801dc471511fe8969a Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Tue, 1 Jun 2021 17:20:26 +0100 Subject: [PATCH 2/6] make DATABASE_URL non mandatory, check env var configuration --- README.md | 2 +- src/config/config.ts | 11 ++++++++--- src/types.ts | 2 +- src/utils/db/hub.ts | 3 +-- src/utils/utils.ts | 15 ++++++++++----- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 33c2b5cc..90389db3 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ There's a multitude of settings you can use to control the plugin server. Use th | KAFKA_PRODUCER_MAX_QUEUE_SIZE | Kafka producer batch max size before flushing | `20` | | KAFKA_FLUSH_FREQUENCY_MS | Kafka producer batch max duration before flushing | `500` | | KAFKA_MAX_MESSAGE_BATCH_SIZE | Kafka producer batch max size in bytes before flushing | `900000` | -| LOG_LEVEL | minimum log level | `LogLevel.Info` | +| LOG_LEVEL | minimum log level | `info` | | SENTRY_DSN | Sentry ingestion URL | `null` | | STATSD_HOST | StatsD host - integration disabled if this is not provided | `null` | | STATSD_PORT | StatsD port | `8125` | diff --git a/src/config/config.ts b/src/config/config.ts index a0ae12dd..0219953e 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -1,19 +1,24 @@ import os from 'os' import { LogLevel, PluginsServerConfig } from '../types' -import { stringToBoolean } from '../utils/utils' +import { determineNodeEnv, NodeEnv, stringToBoolean } from '../utils/utils' import { KAFKA_EVENTS_PLUGIN_INGESTION } from './kafka-topics' export const defaultConfig = overrideWithEnv(getDefaultConfig()) export const configHelp = getConfigHelp() export function getDefaultConfig(): PluginsServerConfig { - const isTestEnv = process.env.NODE_ENV === 'test' + const isTestEnv = determineNodeEnv() === NodeEnv.Test + const isDevEnv = determineNodeEnv() === NodeEnv.Development const coreCount = os.cpus().length return { CELERY_DEFAULT_QUEUE: 'celery', - DATABASE_URL: isTestEnv ? 'postgres://localhost:5432/test_posthog' : 'postgres://localhost:5432/posthog', + DATABASE_URL: isTestEnv + ? 'postgres://localhost:5432/test_posthog' + : isDevEnv + ? 'postgres://localhost:5432/posthog' + : null, POSTHOG_DB_NAME: null, POSTHOG_DB_USER: 'postgres', POSTHOG_DB_PASSWORD: '', diff --git a/src/types.ts b/src/types.ts index 6c1377fb..c6421593 100644 --- a/src/types.ts +++ b/src/types.ts @@ -39,7 +39,7 @@ export interface PluginsServerConfig extends Record { TASKS_PER_WORKER: number TASK_TIMEOUT: number CELERY_DEFAULT_QUEUE: string - DATABASE_URL: string + DATABASE_URL: string | null POSTHOG_DB_NAME: string | null POSTHOG_DB_USER: string POSTHOG_DB_PASSWORD: string diff --git a/src/utils/db/hub.ts b/src/utils/db/hub.ts index bc684272..9e7b50f4 100644 --- a/src/utils/db/hub.ts +++ b/src/utils/db/hub.ts @@ -135,8 +135,7 @@ export async function createHub( timeStr ? DateTime.fromSQL(timeStr, { zone: 'utc' }).toISO() : null ) - const configOrDatabaseUrl = serverConfig.DATABASE_URL ? serverConfig.DATABASE_URL : serverConfig - const postgres = createPostgresPool(configOrDatabaseUrl) + const postgres = createPostgresPool(serverConfig) const redisPool = createPool( { diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 4fe474ae..59cc0ad5 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -416,22 +416,27 @@ export function createPostgresPool( configOrDatabaseUrl: PluginsServerConfig | string, onError?: (error: Error) => any ): Pool { + if (typeof configOrDatabaseUrl !== 'string') { + if (!configOrDatabaseUrl.DATABASE_URL && !configOrDatabaseUrl.POSTHOG_DB_NAME) { + throw new Error('Invalid configuration for Postgres: either DATABASE_URL or POSTHOG_DB_NAME required') + } + } const credentials: Partial = typeof configOrDatabaseUrl === 'string' ? { connectionString: configOrDatabaseUrl, } - : configOrDatabaseUrl.POSTHOG_DB_NAME + : configOrDatabaseUrl.DATABASE_URL ? { - database: configOrDatabaseUrl.POSTHOG_DB_NAME, + connectionString: configOrDatabaseUrl.DATABASE_URL, + } + : { + database: configOrDatabaseUrl.POSTHOG_DB_NAME || undefined, user: configOrDatabaseUrl.POSTHOG_DB_USER, password: configOrDatabaseUrl.POSTHOG_DB_PASSWORD, host: configOrDatabaseUrl.POSTHOG_POSTGRES_HOST, port: configOrDatabaseUrl.POSTHOG_POSTGRES_PORT, } - : { - connectionString: configOrDatabaseUrl.DATABASE_URL, - } const pgPool = new Pool({ ...credentials, From 2f8bafde30bc62158c2ba576d6e46af0c1867612 Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Tue, 1 Jun 2021 17:31:31 +0100 Subject: [PATCH 3/6] fix tests --- tests/helpers/graphile.ts | 2 +- tests/helpers/sql.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/helpers/graphile.ts b/tests/helpers/graphile.ts index e1f220a1..57dc45cb 100644 --- a/tests/helpers/graphile.ts +++ b/tests/helpers/graphile.ts @@ -4,7 +4,7 @@ import { Pool } from 'pg' import { PluginsServerConfig } from '../../src/types' export async function resetGraphileSchema(serverConfig: PluginsServerConfig): Promise { - const graphileUrl = serverConfig.JOB_QUEUE_GRAPHILE_URL || serverConfig.DATABASE_URL + const graphileUrl = serverConfig.JOB_QUEUE_GRAPHILE_URL || serverConfig.DATABASE_URL || undefined const db = new Pool({ connectionString: graphileUrl }) try { diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index e4772d02..bd9b4ac9 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -33,7 +33,7 @@ export async function resetTestDatabase( extraRows: ExtraDatabaseRows = {} ): Promise { const config = { ...defaultConfig, ...extraServerConfig } - const db = new Pool({ connectionString: config.DATABASE_URL }) + const db = new Pool({ connectionString: config.DATABASE_URL || undefined }) try { await db.query('DELETE FROM ee_hook') } catch {} @@ -224,7 +224,7 @@ export function onQuery(hub: Hub, onQueryCallback: (queryText: string) => any): } export async function getErrorForPluginConfig(id: number): Promise { - const db = new Pool({ connectionString: defaultConfig.DATABASE_URL }) + const db = new Pool({ connectionString: defaultConfig.DATABASE_URL || undefined }) let error try { const response = await db.query('SELECT * FROM posthog_pluginconfig WHERE id = $1', [id]) From 4420bd3ff2ba1cb7aa6ee788d6940fe9e0f3ce48 Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Thu, 3 Jun 2021 13:12:52 +0100 Subject: [PATCH 4/6] address comments --- README.md | 2 +- src/utils/utils.ts | 2 +- tests/helpers/graphile.ts | 2 +- tests/helpers/sql.ts | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 90389db3..917a0d98 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ There's a multitude of settings you can use to control the plugin server. Use th | KAFKA_PRODUCER_MAX_QUEUE_SIZE | Kafka producer batch max size before flushing | `20` | | KAFKA_FLUSH_FREQUENCY_MS | Kafka producer batch max duration before flushing | `500` | | KAFKA_MAX_MESSAGE_BATCH_SIZE | Kafka producer batch max size in bytes before flushing | `900000` | -| LOG_LEVEL | minimum log level | `info` | +| LOG_LEVEL | minimum log level | `'info'` | | SENTRY_DSN | Sentry ingestion URL | `null` | | STATSD_HOST | StatsD host - integration disabled if this is not provided | `null` | | STATSD_PORT | StatsD port | `8125` | diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 59cc0ad5..1aee2120 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -431,7 +431,7 @@ export function createPostgresPool( connectionString: configOrDatabaseUrl.DATABASE_URL, } : { - database: configOrDatabaseUrl.POSTHOG_DB_NAME || undefined, + database: configOrDatabaseUrl.POSTHOG_DB_NAME ?? undefined, user: configOrDatabaseUrl.POSTHOG_DB_USER, password: configOrDatabaseUrl.POSTHOG_DB_PASSWORD, host: configOrDatabaseUrl.POSTHOG_POSTGRES_HOST, diff --git a/tests/helpers/graphile.ts b/tests/helpers/graphile.ts index 57dc45cb..6077c214 100644 --- a/tests/helpers/graphile.ts +++ b/tests/helpers/graphile.ts @@ -4,7 +4,7 @@ import { Pool } from 'pg' import { PluginsServerConfig } from '../../src/types' export async function resetGraphileSchema(serverConfig: PluginsServerConfig): Promise { - const graphileUrl = serverConfig.JOB_QUEUE_GRAPHILE_URL || serverConfig.DATABASE_URL || undefined + const graphileUrl = serverConfig.JOB_QUEUE_GRAPHILE_URL || serverConfig.DATABASE_URL! const db = new Pool({ connectionString: graphileUrl }) try { diff --git a/tests/helpers/sql.ts b/tests/helpers/sql.ts index bd9b4ac9..8a26ec7a 100644 --- a/tests/helpers/sql.ts +++ b/tests/helpers/sql.ts @@ -33,7 +33,7 @@ export async function resetTestDatabase( extraRows: ExtraDatabaseRows = {} ): Promise { const config = { ...defaultConfig, ...extraServerConfig } - const db = new Pool({ connectionString: config.DATABASE_URL || undefined }) + const db = new Pool({ connectionString: config.DATABASE_URL! }) try { await db.query('DELETE FROM ee_hook') } catch {} @@ -224,7 +224,7 @@ export function onQuery(hub: Hub, onQueryCallback: (queryText: string) => any): } export async function getErrorForPluginConfig(id: number): Promise { - const db = new Pool({ connectionString: defaultConfig.DATABASE_URL || undefined }) + const db = new Pool({ connectionString: defaultConfig.DATABASE_URL! }) let error try { const response = await db.query('SELECT * FROM posthog_pluginconfig WHERE id = $1', [id]) From 365d3488332f0d74a9e6eae394f57a425add43d6 Mon Sep 17 00:00:00 2001 From: Neil Kakkar Date: Thu, 3 Jun 2021 14:20:18 +0100 Subject: [PATCH 5/6] check what went wrong --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab3f3a05..0f391658 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,8 @@ jobs: echo "⏳ Waiting 10 seconds to see if it runs" sleep 10 + cat tmp/plugin-server.log + echo "🤔 Checking if it's still running, as it should be" if ! kill $pid > /dev/null 2>&1; then echo "😵 It was not running!" From 93a51d6151a4840ddc490cf0e5729bad4382dffa Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Fri, 4 Jun 2021 22:06:34 +0200 Subject: [PATCH 6/6] Remove extra cat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 😾 --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f391658..ab3f3a05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,8 +118,6 @@ jobs: echo "⏳ Waiting 10 seconds to see if it runs" sleep 10 - cat tmp/plugin-server.log - echo "🤔 Checking if it's still running, as it should be" if ! kill $pid > /dev/null 2>&1; then echo "😵 It was not running!"