From 775d807133deaf766d24504e75c98035dfe95d0b Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Wed, 9 Mar 2022 01:19:00 +0530 Subject: [PATCH 1/6] feat: welcome postgres 12! --- .github/workflows/pythontest.yml | 2 +- README.md | 2 +- docker-compose.yml | 2 +- docs/manual_setup.md | 14 ++++++-------- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 8544989bf6..096aae7269 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -24,7 +24,7 @@ jobs: # Label used to access the service container postgres: # Docker Hub image - image: postgres + image: postgres:12.10 # Provide the password for postgres env: POSTGRES_USER: learningequality diff --git a/README.md b/README.md index 4cb7608a0f..28b74caf5a 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ To confirm that the services are running, run `docker ps`, and you should see th CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES e09c5c203b93 redis:4.0.9 "docker-entrypoint.s…" 51 seconds ago Up 49 seconds 0.0.0.0:6379->6379/tcp studio_vue-refactor_redis_1 6164371efb6b minio/minio "minio server /data" 51 seconds ago Up 49 seconds 0.0.0.0:9000->9000/tcp studio_vue-refactor_minio_1 -c86bbfa3a59e postgres:9.6 "docker-entrypoint.s…" 51 seconds ago Up 49 seconds 0.0.0.0:5432->5432/tcp studio_vue-refactor_postgres_1 +c86bbfa3a59e postgres:12.10 "docker-entrypoint.s…" 51 seconds ago Up 49 seconds 0.0.0.0:5432->5432/tcp studio_vue-refactor_postgres_1 ``` diff --git a/docker-compose.yml b/docker-compose.yml index cb1cc9decb..8b2bb69209 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,7 +53,7 @@ services: - minio_data:/data postgres: - image: postgres:9.6 + image: postgres:12.10 environment: PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_USER: learningequality diff --git a/docs/manual_setup.md b/docs/manual_setup.md index c61e0cc082..0db63b8a65 100644 --- a/docs/manual_setup.md +++ b/docs/manual_setup.md @@ -1,5 +1,3 @@ - - # Manually installing dependencies ## Install @@ -9,7 +7,7 @@ Rather than using Docker, it is also possible to manually install the dependenci - [python (2.7)](https://www.python.org/downloads/release/python-2713/) - [python-pip](https://pip.pypa.io/en/stable/installing/) - [nodejs (10.x)](https://nodejs.org/en/download/) - - [Postgres DB](https://www.postgresql.org/download/) + - [Postgres DB (12.10)](https://www.postgresql.org/download/) - [redis](https://redis.io/topics/quickstart) - [minio server](https://www.minio.io/downloads.html) - [nginx](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/) @@ -37,7 +35,7 @@ curl -sL https://deb.nodesource.com/setup_10.x | bash - # Install packages apt-get install -y python python-pip python-dev python-tk \ - postgresql-server-dev-all postgresql-contrib postgresql-client postgresql \ + postgresql-server-dev-all postgresql-contrib postgresql-client postgresql-12.10 \ ffmpeg nodejs libmagickwand-dev nginx redis-server wkhtmltopdf ``` @@ -46,9 +44,9 @@ apt-get install -y python python-pip python-dev python-tk \ You can install the corresponding packages using Homebrew: ```bash -brew install postgresql@9.6 redis node ffmpeg imagemagick@6 gs +brew install postgresql@12.10 redis node ffmpeg imagemagick@6 gs brew install minio/stable/minio -brew link --force postgresql@9.6 +brew link --force postgresql@12.10 brew link --force imagemagick@6 ``` @@ -62,13 +60,13 @@ Windows is no longer supported due to incompatibilities with some of the require ## Set up the database -Install [postgres](https://www.postgresql.org/download/) if you don't have it already. If you're using a package manager, you need to make sure you install the following packages: `postgresql`, `postgresql-contrib`, and `postgresql-server-dev-all` which will be required to build `psycopg2` python driver. +Install [postgres](https://www.postgresql.org/download/) if you don't have it already. If you're using a package manager, you need to make sure you install the following packages: `postgresql-12.10`, `postgresql-contrib`, and `postgresql-server-dev-all` which will be required to build `psycopg2` python driver. Make sure postgres is running: ```bash service postgresql start -# alternatively: pg_ctl -D /usr/local/var/postgresql@9.6 start +# alternatively: pg_ctl -D /usr/local/var/postgresql@12.10 start ``` Start the client with: From d617233b5a56cc7550295a7ef9b260bbf54ca208 Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Wed, 9 Mar 2022 02:12:16 +0530 Subject: [PATCH 2/6] Pin CI services version to same as docker-compose services --- .github/workflows/pythontest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 096aae7269..71f1663235 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -42,7 +42,7 @@ jobs: # Label used to access the service container redis: # Docker Hub image - image: redis + image: redis:4.0.9 # Set health checks to wait until redis has started options: >- --health-cmd "redis-cli ping" @@ -53,7 +53,7 @@ jobs: # Maps port 6379 on service container to the host - 6379:6379 minio: - image: bitnami/minio + image: minio/minio:RELEASE.2020-06-22T03-12-50Z env: MINIO_API_CORS_ALLOW_ORIGIN: http://localhost:8080 MINIO_ACCESS_KEY: development From c2d8a6fec50403bf27c75dfb63be60ee9cbd387b Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 10 Mar 2022 23:22:57 +0530 Subject: [PATCH 3/6] fix: pin postgres major & use minio/minio on CI --- .github/workflows/pythontest.yml | 31 ++++++++++++++++++++++--------- docker-compose.yml | 2 +- docs/manual_setup.md | 12 ++++++------ 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 71f1663235..6c4f1f3aa7 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -15,16 +15,18 @@ jobs: with: github_token: ${{ github.token }} paths: '["**.py", "requirements.txt", "requirements-dev.txt", ".github/workflows/pythontest.yml"]' + unit_test: name: Python unit tests needs: pre_job if: ${{ needs.pre_job.outputs.should_skip != 'true' }} runs-on: ubuntu-latest + services: # Label used to access the service container postgres: # Docker Hub image - image: postgres:12.10 + image: postgres:12 # Provide the password for postgres env: POSTGRES_USER: learningequality @@ -52,16 +54,22 @@ jobs: ports: # Maps port 6379 on service container to the host - 6379:6379 - minio: - image: minio/minio:RELEASE.2020-06-22T03-12-50Z - env: - MINIO_API_CORS_ALLOW_ORIGIN: http://localhost:8080 - MINIO_ACCESS_KEY: development - MINIO_SECRET_KEY: development - ports: - - 9000:9000 steps: - uses: actions/checkout@v2 + - name: Set up minio + run: | + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ACCESS_KEY=minioadmin" \ + -e "MINIO_SECRET_KEY=minioadmin" \ + -v /tmp/minio_data:/data \ + -v /tmp/minio_config:/root/.minio \ + minio/minio server /data + + export AWS_ACCESS_KEY_ID=minioadmin + export AWS_SECRET_ACCESS_KEY=minioadmin + export AWS_EC2_METADATA_DISABLED=true + + aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket - name: Set up Python 3.6 uses: actions/setup-python@v2 with: @@ -79,6 +87,11 @@ jobs: pip install pip-tools pip-sync requirements.txt requirements-dev.txt - name: Test pytest + env: + AWS_BUCKET_NAME: testbucket + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + AWS_S3_ENDPOINT_URL: http://127.0.0.1:9000 run: | sh -c './contentcuration/manage.py makemigrations --check' pytest diff --git a/docker-compose.yml b/docker-compose.yml index 8b2bb69209..02732ba585 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,7 +53,7 @@ services: - minio_data:/data postgres: - image: postgres:12.10 + image: postgres:12 environment: PGDATA: /var/lib/postgresql/data/pgdata POSTGRES_USER: learningequality diff --git a/docs/manual_setup.md b/docs/manual_setup.md index 0db63b8a65..f0e3c77cf7 100644 --- a/docs/manual_setup.md +++ b/docs/manual_setup.md @@ -7,7 +7,7 @@ Rather than using Docker, it is also possible to manually install the dependenci - [python (2.7)](https://www.python.org/downloads/release/python-2713/) - [python-pip](https://pip.pypa.io/en/stable/installing/) - [nodejs (10.x)](https://nodejs.org/en/download/) - - [Postgres DB (12.10)](https://www.postgresql.org/download/) + - [Postgres DB (12.x)](https://www.postgresql.org/download/) - [redis](https://redis.io/topics/quickstart) - [minio server](https://www.minio.io/downloads.html) - [nginx](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/) @@ -35,7 +35,7 @@ curl -sL https://deb.nodesource.com/setup_10.x | bash - # Install packages apt-get install -y python python-pip python-dev python-tk \ - postgresql-server-dev-all postgresql-contrib postgresql-client postgresql-12.10 \ + postgresql-server-dev-all postgresql-contrib postgresql-client postgresql-12 \ ffmpeg nodejs libmagickwand-dev nginx redis-server wkhtmltopdf ``` @@ -44,9 +44,9 @@ apt-get install -y python python-pip python-dev python-tk \ You can install the corresponding packages using Homebrew: ```bash -brew install postgresql@12.10 redis node ffmpeg imagemagick@6 gs +brew install postgresql@12 redis node ffmpeg imagemagick@6 gs brew install minio/stable/minio -brew link --force postgresql@12.10 +brew link --force postgresql@12 brew link --force imagemagick@6 ``` @@ -60,13 +60,13 @@ Windows is no longer supported due to incompatibilities with some of the require ## Set up the database -Install [postgres](https://www.postgresql.org/download/) if you don't have it already. If you're using a package manager, you need to make sure you install the following packages: `postgresql-12.10`, `postgresql-contrib`, and `postgresql-server-dev-all` which will be required to build `psycopg2` python driver. +Install [postgres](https://www.postgresql.org/download/) if you don't have it already. If you're using a package manager, you need to make sure you install the following packages: `postgresql-12`, `postgresql-contrib`, and `postgresql-server-dev-all` which will be required to build `psycopg2` python driver. Make sure postgres is running: ```bash service postgresql start -# alternatively: pg_ctl -D /usr/local/var/postgresql@12.10 start +# alternatively: pg_ctl -D /usr/local/var/postgresql@12 start ``` Start the client with: From abbf18d81f5d7db16257c2f98379637b7449c88d Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 10 Mar 2022 23:28:19 +0530 Subject: [PATCH 4/6] fix: unnecessary exports removed from minio CI --- .github/workflows/pythontest.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 6c4f1f3aa7..9b26f09ba8 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -64,11 +64,6 @@ jobs: -v /tmp/minio_data:/data \ -v /tmp/minio_config:/root/.minio \ minio/minio server /data - - export AWS_ACCESS_KEY_ID=minioadmin - export AWS_SECRET_ACCESS_KEY=minioadmin - export AWS_EC2_METADATA_DISABLED=true - aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket - name: Set up Python 3.6 uses: actions/setup-python@v2 From cd50e93a1845df707d32115e85ec511225b9ae9a Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 10 Mar 2022 23:31:35 +0530 Subject: [PATCH 5/6] fix: CI --- .github/workflows/pythontest.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 9b26f09ba8..966dd9613f 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -58,13 +58,16 @@ jobs: - uses: actions/checkout@v2 - name: Set up minio run: | - docker run -d -p 9000:9000 --name minio \ - -e "MINIO_ACCESS_KEY=minioadmin" \ - -e "MINIO_SECRET_KEY=minioadmin" \ - -v /tmp/minio_data:/data \ - -v /tmp/minio_config:/root/.minio \ - minio/minio server /data - aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ACCESS_KEY=minioadmin" \ + -e "MINIO_SECRET_KEY=minioadmin" \ + -v /tmp/minio_data:/data \ + -v /tmp/minio_config:/root/.minio \ + minio/minio server /data + export AWS_ACCESS_KEY_ID=minioadmin + export AWS_SECRET_ACCESS_KEY=minioadmin + export AWS_EC2_METADATA_DISABLED=true + aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket - name: Set up Python 3.6 uses: actions/setup-python@v2 with: From 9c8e33f9e41de925efc44204cdd96d90be8683a0 Mon Sep 17 00:00:00 2001 From: Vivek Agrawal Date: Thu, 10 Mar 2022 23:35:55 +0530 Subject: [PATCH 6/6] fix: Blaine's elegant solution! --- .github/workflows/pythontest.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pythontest.yml b/.github/workflows/pythontest.yml index 966dd9613f..7fdf6e178c 100644 --- a/.github/workflows/pythontest.yml +++ b/.github/workflows/pythontest.yml @@ -59,15 +59,11 @@ jobs: - name: Set up minio run: | docker run -d -p 9000:9000 --name minio \ - -e "MINIO_ACCESS_KEY=minioadmin" \ - -e "MINIO_SECRET_KEY=minioadmin" \ + -e "MINIO_ACCESS_KEY=development" \ + -e "MINIO_SECRET_KEY=development" \ -v /tmp/minio_data:/data \ -v /tmp/minio_config:/root/.minio \ minio/minio server /data - export AWS_ACCESS_KEY_ID=minioadmin - export AWS_SECRET_ACCESS_KEY=minioadmin - export AWS_EC2_METADATA_DISABLED=true - aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket - name: Set up Python 3.6 uses: actions/setup-python@v2 with: @@ -85,11 +81,6 @@ jobs: pip install pip-tools pip-sync requirements.txt requirements-dev.txt - name: Test pytest - env: - AWS_BUCKET_NAME: testbucket - AWS_ACCESS_KEY_ID: minioadmin - AWS_SECRET_ACCESS_KEY: minioadmin - AWS_S3_ENDPOINT_URL: http://127.0.0.1:9000 run: | sh -c './contentcuration/manage.py makemigrations --check' pytest