Skip to content

Commit 59655a3

Browse files
GracefulLemmingGracefulLemming
authored
Merge branch 'main' into cm/safari-audio-hack
2 parents e13977f + ec4c8fe commit 59655a3

32 files changed

+43927
-2490
lines changed

‎.git-hooks/pre-commit

100755100644
+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ if git-staged "website/src/graphql/**.graphql" && ! git-staged "website/src/grap
2626
fi
2727

2828
# Reqire frontend formatting
29-
if git-staged "website/*" && cd website && yarn prettier -c --config package.json src; then
29+
if git-staged "website/*" && cd website && ! yarn prettier -c --config package.json src; then
3030
printf "Frontend code edited but not formatted.\nPlease run 'dev-check' \nor\n'cd website; prettier -w --config package.json src'.\n"
3131
HOOKS_FAILED=1
3232
fi

‎.github/workflows/main.yml

+19-33
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@ concurrency:
2020
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
2121
jobs:
2222
build:
23-
runs-on: ubuntu-20.04
23+
runs-on: ubuntu-22.04
2424
environment: ${{ github.event_name == 'release' && ( github.event.action == 'released' && 'Production' || 'Development') || 'Development' }}
2525
env:
26-
DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
26+
DATABASE_PASSWORD: ${{ github.event.action == 'prereleased' && secrets.UAT_DATABASE_PASSWORD || secrets.DATABASE_PASSWORD }}
2727
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
2828
AWS_VPC_ID: ${{ secrets.AWS_VPC_ID }}
2929
AWS_SUBNET_PRIMARY: ${{ secrets.AWS_SUBNET_PRIMARY }}
@@ -52,7 +52,7 @@ jobs:
5252
branch: ${{ github.event.action == 'prereleased' && 'uat' || 'release' }}
5353
force: true
5454
- name: Configure AWS credentials
55-
uses: aws-actions/configure-aws-credentials@v2
55+
uses: aws-actions/configure-aws-credentials@v4
5656
with:
5757
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
5858
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -88,48 +88,34 @@ jobs:
8888
pushFilter: "(-dailp$|-dailp-|-terraform-config$|-source$|\\.tar\\.gz$|-output$|-plan$|-apply-now$|-apply$)"
8989
- name: Build and test project
9090
# nix -L argument shows the full build log and --impure allows it to access environment variables.
91+
env:
92+
BASTION_ID: ${{ (env.TF_STAGE == 'prod' && secrets.EC2_INSTANCE ) || (env.TF_STAGE == 'uat' && secrets.UAT_EC2_INSTANCE) || secrets.DEV_EC2_INSTANCE }}
9193
run: |
9294
nix build --impure -L
9395
- name: Deploy back-end to AWS via terraform
96+
env:
97+
BASTION_ID: ${{ (env.TF_STAGE == 'prod' && secrets.EC2_INSTANCE ) || (env.TF_STAGE == 'uat' && secrets.UAT_EC2_INSTANCE) || secrets.DEV_EC2_INSTANCE }}
9498
run: |
9599
nix run --impure -L .#tf-apply-now
96100
SECURITY_GROUP_ID=$(nix run --impure .#tf-output access_security_group_id)
97101
echo "ACCESS_SECURITY_GROUP=$SECURITY_GROUP_ID" >> $GITHUB_ENV
98-
# - name: Validate spreadsheets
99-
# env:
100-
# RUST_LOG: warn
101-
# run: nix run --impure -L .#validate-data
102-
- name: Allow SSH access to bastion host
103-
uses: sohelamin/aws-security-group-add-ip-action@master
104-
with:
105-
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
106-
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
107-
aws-region: us-east-1
108-
aws-security-group-id: ${{ env.ACCESS_SECURITY_GROUP }}
109-
port: 22
110-
to-port: 22
111-
protocol: tcp
112-
- name: Encode documents as TEI and database entries
113-
# Sets up port forwarding to remote database on port 5432
102+
- name: Update database schema
103+
env:
104+
BASTION_IP: ${{ (env.TF_STAGE == 'prod' && secrets.PROD_BASTION_IP ) || (env.TF_STAGE == 'uat' && secrets.UAT_BASTION_IP) || secrets.DEV_BASTION_IP }}
105+
BASTION_ID: ${{ (env.TF_STAGE == 'prod' && secrets.EC2_INSTANCE ) || (env.TF_STAGE == 'uat' && secrets.UAT_EC2_INSTANCE) || secrets.DEV_EC2_INSTANCE }}
114106
run: |
115-
echo "Creating SSH key file..."
116-
echo "${{secrets.AWS_BASTION_SSH_KEY}}" > dailp-deployment-key.pem
117-
chmod 400 ./dailp-deployment-key.pem
118107
echo "Retrieving terraform outputs..."
119-
BASTION_IP=$(nix run --impure .#tf-output bastion_ip)
120-
DATABASE_ENDPOINT=$(nix run --impure .#tf-output database_endpoint)
121-
echo "Configuring SSH client..."
122-
mkdir -p ~/.ssh
123-
echo "Still configuring SSH client..."
124-
ssh-keyscan -H $BASTION_IP >> ~/.ssh/known_hosts
125-
echo "Forwarding port 5432 to remote database $DATABASE_ENDPOINT through $BASTION_IP"
126-
ssh -i ./dailp-deployment-key.pem -f -N -L 5432:$DATABASE_ENDPOINT ec2-user@$BASTION_IP
127-
108+
DATABASE_ADDRESS=$(nix run --impure .#tf-output database_address)
109+
echo "Connecting to bastion via SSM..."
110+
aws ssm start-session \
111+
--target ${{ env.BASTION_ID }} \
112+
--document-name AWS-StartPortForwardingSessionToRemoteHost \
113+
--parameters '{"host":[ '"\"$DATABASE_ADDRESS\""' ],"portNumber":["5432"], "localPortNumber":["5432"]}' &
114+
echo "Updating shell variables..."
128115
export DAILP_API_URL=$(nix run --impure .#tf-output functions_url)
129116
export DATABASE_URL=postgres://dailp:$DATABASE_PASSWORD@localhost:5432/dailp
117+
echo "Migrating schema..."
130118
nix run --impure .#migrate-schema
131-
export CF_URL=$(nix run --impure .#tf-output cloudfront_distro_url)
132-
# nix run --impure .#migrate-data
133119
- name: Publish website
134120
run: |
135121
curl -X POST -d {} "$(nix run --impure .#tf-output amplify_webhook)" -H "Content-Type:application/json"

‎.github/workflows/migrate-data.yml

+24-26
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@ on:
1313
- Production
1414
jobs:
1515
build:
16-
runs-on: ubuntu-20.04
16+
runs-on: ubuntu-22.04
1717
environment: ${{ inputs.applyTo == 'Production' && 'Production' || 'Development'}}
1818
env:
19-
DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
19+
DATABASE_PASSWORD: ${{ github.event.action == 'prereleased' && secrets.UAT_DATABASE_PASSWORD || secrets.DATABASE_PASSWORD }}
2020
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
2121
AWS_VPC_ID: ${{ secrets.AWS_VPC_ID }}
2222
AWS_SUBNET_PRIMARY: ${{ secrets.AWS_SUBNET_PRIMARY }}
@@ -29,6 +29,7 @@ jobs:
2929
GIT_REPOSITORY_URL: https://github.com/neu-dsg/dailp-encoding
3030
OAUTH_TOKEN: ${{ secrets.OAUTH_TOKEN }}
3131
RUST_LOG: info
32+
BASTION_IP: ${{ secrets.TEST_BASTION_IP }}
3233
TF_STAGE: ${{ inputs.applyTo == 'Development' && 'dev' || ( inputs.applyTo == 'Staging/UAT' && 'uat' || 'prod' )}}
3334
steps:
3435
- name: Checkout code
@@ -56,36 +57,33 @@ jobs:
5657
nix run --impure -L .#tf-init
5758
SECURITY_GROUP_ID=$(nix run --impure .#tf-output access_security_group_id)
5859
echo "ACCESS_SECURITY_GROUP=$SECURITY_GROUP_ID" >> $GITHUB_ENV
59-
# - name: Validate spreadsheets
60-
# run: nix run --impure -L .#validate-data
61-
- name: Add public IP to AWS security group
62-
uses: sohelamin/aws-security-group-add-ip-action@master
63-
with:
64-
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
65-
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
66-
aws-region: us-east-1
67-
aws-security-group-id: ${{ env.ACCESS_SECURITY_GROUP }}
68-
port: 22
69-
to-port: 22
70-
protocol: tcp
60+
# - name: Add public IP to AWS security group
61+
# uses: sohelamin/aws-security-group-add-ip-action@master
62+
# with:
63+
# aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
64+
# aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
65+
# aws-region: us-east-1
66+
# aws-security-group-id: ${{ env.ACCESS_SECURITY_GROUP }}
67+
# port: 22
68+
# to-port: 22
69+
# protocol: tcp
7170
- name: Migrate from spreadsheets to database
72-
# Port forward 5432 to remote database
71+
env:
72+
BASTION_IP: ${{ (env.TF_STAGE == 'prod' && secrets.PROD_BASTION_IP ) || (env.TF_STAGE == 'uat' && secrets.UAT_BASTION_IP) || secrets.DEV_BASTION_IP }}
73+
BASTION_ID: ${{ (env.TF_STAGE == 'prod' && secrets.EC2_INSTANCE ) || (env.TF_STAGE == 'uat' && secrets.UAT_EC2_INSTANCE) || secrets.DEV_EC2_INSTANCE }}
7374
run: |
74-
echo "Creating SSH key file..."
75-
echo "${{secrets.AWS_BASTION_SSH_KEY}}" > dailp-deployment-key.pem
76-
chmod 400 dailp-deployment-key.pem
7775
echo "Retrieving terraform outputs..."
78-
BASTION_IP=$(nix run --impure .#tf-output bastion_ip)
79-
DATABASE_ENDPOINT=$(nix run --impure .#tf-output database_endpoint)
80-
echo "Configuring SSH client..."
81-
mkdir -p ~/.ssh
82-
ssh-keyscan -H $BASTION_IP >> ~/.ssh/known_hosts
83-
echo "Forwarding port 5432 to remote database"
84-
ssh -i dailp-deployment-key.pem -f -N -L 5432:$DATABASE_ENDPOINT ec2-user@$BASTION_IP
85-
76+
DATABASE_ADDRESS=$(nix run --impure .#tf-output database_address)
77+
echo "Connecting to bastion via SSM..."
78+
aws ssm start-session \
79+
--target ${{ env.BASTION_ID }} \
80+
--document-name AWS-StartPortForwardingSessionToRemoteHost \
81+
--parameters '{"host":[ '"\"$DATABASE_ADDRESS\""' ],"portNumber":["5432"], "localPortNumber":["5432"]}' &
82+
echo "Updating shell variables..."
8683
export DAILP_API_URL=$(nix run --impure .#tf-output functions_url)
8784
export DATABASE_URL=postgres://dailp:$DATABASE_PASSWORD@localhost:5432/dailp
8885
export CF_URL=$(nix run --impure .#tf-output cloudfront_distro_url)
86+
echo "Updating data..."
8987
nix run --impure -L .#migrate-data
9088
- name: Publish website
9189
run: |

‎.github/workflows/publish-website.yml

+2-3
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@ on:
33
workflow_dispatch:
44
jobs:
55
publish_website:
6-
runs-on: ubuntu-20.04
6+
runs-on: ubuntu-22.04
77
environment: ${{ github.ref == 'refs/heads/main' && 'Development' || 'Production' }}
88
env:
9-
DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
9+
DATABASE_PASSWORD: ${{ github.event.action == 'prereleased' && secrets.UAT_DATABASE_PASSWORD || secrets.DATABASE_PASSWORD }}
1010
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
1111
AWS_VPC_ID: ${{ secrets.AWS_VPC_ID }}
1212
AWS_SUBNET_PRIMARY: ${{ secrets.AWS_SUBNET_PRIMARY }}
@@ -45,7 +45,6 @@ jobs:
4545
# nix -L argument shows the full build log and --impure allows it to access environment variables.
4646
run: |
4747
nix build --impure -L
48-
cp -f ./result/config.tf.json ./config.tf.json
4948
- name: Publish website
5049
run: |
5150
nix run --impure -L .#tf-plan

‎.github/workflows/terraform-plan.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ on:
44
branches: [main]
55
jobs:
66
plan:
7-
runs-on: ubuntu-20.04
7+
runs-on: ubuntu-22.04
88
env:
9-
DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
9+
DATABASE_PASSWORD: ${{ github.event.action == 'prereleased' && secrets.UAT_DATABASE_PASSWORD || secrets.DATABASE_PASSWORD }}
1010
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
1111
AWS_VPC_ID: ${{ secrets.AWS_VPC_ID }}
1212
AWS_SUBNET_PRIMARY: ${{ secrets.AWS_SUBNET_PRIMARY }}

‎check.sh

100755100644
+47-47
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,47 @@
1-
#!/usr/bin/env bash
2-
3-
cd $PROJECT_ROOT
4-
5-
echo "--- DATABASE ---"
6-
echo "Checking if SQL queries are prepared..."
7-
if [ ! -d "types/.sqlx" ] || [ ! -n "$(ls -A "types/.sqlx")" ]; then
8-
echo "Prepared queries not present. run 'dev-generate-types'."
9-
exit 1
10-
fi
11-
echo "Checking if SQL types are up to date..."
12-
if $(cd types && cargo sqlx prepare --check -- -p dailp &>/dev/null); then
13-
echo "Generating SQL types..."
14-
cd types
15-
cargo sqlx prepare -- -p dailp &>/dev/null || exit 1
16-
cd $PROJECT_ROOT
17-
fi
18-
19-
echo "--- SERVER ---"
20-
echo "Formatting Rust code..."
21-
cargo fmt &>/dev/null
22-
echo "Checking back-end for errors..."
23-
export SQLX_OFFLINE=true
24-
cargo check &>/dev/null ||
25-
(echo "Back-end build failed, run 'cargo check' for details." && exit 1)
26-
27-
echo "Generating GraphQL schema..."
28-
cargo run --bin dailp-graphql-schema &>/dev/null ||
29-
(echo "GraphQL server build failed, run 'cargo check' for details." && exit 1)
30-
31-
echo "--- WEBSITE ---"
32-
cd website
33-
echo "Formatting TypeScript code..."
34-
yarn prettier --write --config package.json src &>/dev/null
35-
echo "Generating Typescript types for GraphQL queries..."
36-
yarn generate
37-
echo "Checking website for errors..."
38-
yarn tsc || exit 1
39-
40-
echo "--- FINAL CHECKS ---"
41-
cd $PROJECT_ROOT
42-
echo "Checking documentation and formatting..."
43-
./.git-hooks/pre-commit || exit 1
44-
45-
echo
46-
echo "--- NEXT STEPS ---"
47-
echo "Please stage relevant automatic changes"
1+
#!/usr/bin/env bash
2+
3+
cd $PROJECT_ROOT
4+
5+
echo "--- DATABASE ---"
6+
echo "Checking if SQL queries are prepared..."
7+
if [ ! -d "types/.sqlx" ] || [ ! -n "$(ls -A "types/.sqlx")" ]; then
8+
echo "Prepared queries not present. run 'dev-generate-types'."
9+
exit 1
10+
fi
11+
echo "Checking if SQL types are up to date..."
12+
if $(cd types && cargo sqlx prepare --check -- -p dailp &>/dev/null); then
13+
echo "Generating SQL types..."
14+
cd types
15+
cargo sqlx prepare -- -p dailp &>/dev/null || exit 1
16+
cd $PROJECT_ROOT
17+
fi
18+
19+
echo "--- SERVER ---"
20+
echo "Formatting Rust code..."
21+
cargo fmt &>/dev/null
22+
echo "Checking back-end for errors..."
23+
export SQLX_OFFLINE=true
24+
cargo check &>/dev/null ||
25+
(echo "Back-end build failed, run 'cargo check' for details." && exit 1)
26+
27+
echo "Generating GraphQL schema..."
28+
cargo run --bin dailp-graphql-schema &>/dev/null ||
29+
(echo "GraphQL server build failed, run 'cargo check' for details." && exit 1)
30+
31+
echo "--- WEBSITE ---"
32+
cd website
33+
echo "Formatting TypeScript code..."
34+
yarn prettier --write --config package.json src &>/dev/null
35+
echo "Generating Typescript types for GraphQL queries..."
36+
yarn generate
37+
echo "Checking website for errors..."
38+
yarn tsc || exit 1
39+
40+
echo "--- FINAL CHECKS ---"
41+
cd $PROJECT_ROOT
42+
echo "Checking documentation and formatting..."
43+
./.git-hooks/pre-commit || exit 1
44+
45+
echo
46+
echo "--- NEXT STEPS ---"
47+
echo "Please stage relevant automatic changes"

0 commit comments

Comments
 (0)