From ba4c7ccf64f8979a7192602e6db0ded6749fa20f Mon Sep 17 00:00:00 2001 From: Nick Bland Date: Mon, 18 Mar 2024 13:56:21 +1000 Subject: [PATCH] Complete Chapter 5 + Add in More CI/CD + Update Tests and Configurations --- .dockerignore | 6 ++ .gitlab-ci.yml | 77 ++++++++------- ...9be78b46d703924482cf8b43842705fcb7714.json | 17 ++++ Cargo.lock | 12 +++ Cargo.toml | 1 + Dockerfile | 32 ++++++ configuration.yaml | 7 -- configuration/base.yaml | 8 ++ configuration/local.yaml | 4 + configuration/production.yaml | 4 + scripts/remove_test_dbs.sh | 14 +++ src/configuration.rs | 97 +++++++++++++++---- src/main.rs | 13 ++- tests/health_check.rs | 7 +- 14 files changed, 225 insertions(+), 74 deletions(-) create mode 100644 .dockerignore create mode 100644 .sqlx/query-bcfcfebc6f5e8ffbf97d97c5a209be78b46d703924482cf8b43842705fcb7714.json create mode 100644 Dockerfile delete mode 100644 configuration.yaml create mode 100644 configuration/base.yaml create mode 100644 configuration/local.yaml create mode 100644 configuration/production.yaml create mode 100755 scripts/remove_test_dbs.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5d2a7ec --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.env +target/ +tests/ +Dockerfile +scripts/ +migrations/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 90230e8..5618c00 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,50 +1,57 @@ -default: - before_script: - - rustc --version && cargo --version - image: "registry.nickbland.dev/nickbland/rust-docker-ci:latest" - cache: - key: - files: - - Cargo.lock - paths: - - .cargo/ - - target/ - policy: pull-push +image: "rust:latest" + +services: + - postgres:latest variables: - FF_USE_FASTZIP: "true" - CACHE_COMPRESSION_LEVEL: "fastest" + POSTGRES_DB: newsletter + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_HOST: postgres + DB_PORT: 5432 + DATABASE_URL: "postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$DB_PORT/$POSTGRES_DB" + APP_DATABASE__HOST: $POSTGRES_HOST + +cache: # Caches build artifacts so we don't build from scratch in both build and test + key: ${CI_COMMIT_REF_SLUG} + paths: + - .cargo/bin + - .cargo/registry/index + - .cargo/registry/cache + - target/debug/deps + - target/debug/build + policy: pull-push + +default: + before_script: + - export CARGO_HOME="$CI_PROJECT_DIR/.cargo" + - export PATH="$CARGO_HOME/bin:$PATH" + - rustc --version + - cargo --version + - if ! [ -x "$(command -v cargo-sqlx)" ]; then cargo install --version='~0.7' sqlx-cli --no-default-features --features rustls,postgres; fi + - apt update -yq && apt-get install -yq postgresql-client + - SKIP_DOCKER=true ./scripts/init_db.sh + # This is to ensure that the database is reachable and give it some time to initialize. + - until psql "dbname=$POSTGRES_DB user=$POSTGRES_USER password=$POSTGRES_PASSWORD host=postgres" -c '\l'; do sleep 3; done stages: - - format + - build - test -format-code: - stage: format +build: + stage: build script: - - cargo fmt -- --check - -postgres-service: - stage: test - services: - - postgres:15-alpine - variables: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: password - POSTGRES_DB: newsletter - POSTGRES_HOST: postgres - POSTGRES_PORT: 5432 - script: - - SKIP_DOCKER=true ./scripts/init_db.sh # Migrate DB + - cargo build test-code: stage: test - needs: ["postgres-service"] script: - - export DATABASE_URL="postgres://postgres:password@postgres:5432/newsletter" cargo test + - cargo test + - if ! [ -x "$(command -v cargo-tarpaulin)" ]; then cargo install cargo-tarpaulin; fi + - cargo tarpaulin --ignore-tests lint-code: stage: test - needs: ["postgres-service"] script: - - DATABASE_URL="postgres://postgres:password@postgres:5432/newsletter" cargo clippy -- -D warnings \ No newline at end of file + - rustup component add clippy + - cargo clippy -- -D warnings diff --git a/.sqlx/query-bcfcfebc6f5e8ffbf97d97c5a209be78b46d703924482cf8b43842705fcb7714.json b/.sqlx/query-bcfcfebc6f5e8ffbf97d97c5a209be78b46d703924482cf8b43842705fcb7714.json new file mode 100644 index 0000000..0eac09d --- /dev/null +++ b/.sqlx/query-bcfcfebc6f5e8ffbf97d97c5a209be78b46d703924482cf8b43842705fcb7714.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO subscriptions (id, email, name, subscribed_at)\n VALUES ($1, $2, $3, $4)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "bcfcfebc6f5e8ffbf97d97c5a209be78b46d703924482cf8b43842705fcb7714" +} diff --git a/Cargo.lock b/Cargo.lock index b8d8eb8..c7acac7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1164,6 +1164,7 @@ dependencies = [ "reqwest", "secrecy", "serde", + "serde-aux", "sqlx", "tokio", "tracing", @@ -1849,6 +1850,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-aux" +version = "4.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d2e8bfba469d06512e11e3311d4d051a4a387a5b42d010404fecf3200321c95" +dependencies = [ + "chrono", + "serde", + "serde_json", +] + [[package]] name = "serde_derive" version = "1.0.188" diff --git a/Cargo.toml b/Cargo.toml index dc2c5fe..3e2ec0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ name = "mail_app" actix-web = "4" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } serde = { version = "1", features = ["derive"] } +serde-aux = "4" config = { version = "0.13", default-features = false, features = ["yaml"] } uuid = { version = "1", features = ["v4"] } chrono = { version = "0.4.22", default-features = false, features = ["clock"] } diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..94b38f8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +##### Chef +FROM lukemathwalker/cargo-chef:latest-rust-1.76.0 as chef +WORKDIR /app +RUN apt update && apt install lld clang -y + +##### Planner +FROM chef as planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +##### Builder +# Builder prepares project dependancies, not the application. +FROM chef as builder +COPY --from=planner /app/recipe.json recipe.json +RUN cargo chef cook --release --recipe-path recipe.json +COPY . . +ENV SQLX_OFFLINE true +# Now build the application itself. +RUN cargo build --release --bin mail_app + +##### Runtime +FROM debian:bookworm-slim as runtime + +WORKDIR /app +RUN apt update && apt install -y --no-install-recommends openssl ca-certificates \ + && apt autoremove -y \ + && apt clean -y \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /app/target/release/mail_app mail_app +COPY configuration configuration +ENV APP_ENVIRONMENT production +ENTRYPOINT ["./mail_app"] diff --git a/configuration.yaml b/configuration.yaml deleted file mode 100644 index 61a5cb3..0000000 --- a/configuration.yaml +++ /dev/null @@ -1,7 +0,0 @@ -application_port: 8000 -database: - host: "127.0.0.1" - port: 5432 - username: "postgres" - password: "password" - database_name: "newsletter" \ No newline at end of file diff --git a/configuration/base.yaml b/configuration/base.yaml new file mode 100644 index 0000000..a7e4a5d --- /dev/null +++ b/configuration/base.yaml @@ -0,0 +1,8 @@ +application: + port: 8000 +database: + host: "localhost" + port: 5432 + username: "postgres" + password: "password" + database_name: "newsletter" diff --git a/configuration/local.yaml b/configuration/local.yaml new file mode 100644 index 0000000..8fd67fa --- /dev/null +++ b/configuration/local.yaml @@ -0,0 +1,4 @@ +application: + host: 127.0.0.1 +database: + require_ssl: false diff --git a/configuration/production.yaml b/configuration/production.yaml new file mode 100644 index 0000000..cd4608a --- /dev/null +++ b/configuration/production.yaml @@ -0,0 +1,4 @@ +application: + host: 0.0.0.0 +database: + require_ssl: true diff --git a/scripts/remove_test_dbs.sh b/scripts/remove_test_dbs.sh new file mode 100755 index 0000000..5b4eeb5 --- /dev/null +++ b/scripts/remove_test_dbs.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -x +set -eo pipefail + +DB_USER="${POSTGRES_USER:=postgres}" +DB_PASSWORD="${POSTGRES_PASSWORD:=password}" +DB_NAME="${POSTGRES_DB:=newsletter}" +DB_PORT="${POSTGRES_PORT:=5432}" +DB_HOST="${POSTGRES_HOST:=localhost}" + +for dbname in $(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "copy (select datname from pg_database where datname like '%-%-%-%-%') to stdout") ; do + echo "dropping database $dbname" + dropdb -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" "$dbname" +done diff --git a/src/configuration.rs b/src/configuration.rs index 21a9bf0..2eba5a4 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -1,50 +1,105 @@ use secrecy::{ExposeSecret, Secret}; +use serde_aux::field_attributes::deserialize_number_from_string; +use sqlx::postgres::{PgConnectOptions, PgSslMode}; #[derive(serde::Deserialize)] pub struct Settings { pub database: DatabaseSettings, - pub application_port: u16, + pub application: ApplicationSettings, } #[derive(serde::Deserialize)] pub struct DatabaseSettings { pub username: String, pub password: Secret, + #[serde(deserialize_with = "deserialize_number_from_string")] pub port: u16, pub host: String, pub database_name: String, + pub require_ssl: bool, +} + +#[derive(serde::Deserialize)] +pub struct ApplicationSettings { + #[serde(deserialize_with = "deserialize_number_from_string")] + pub port: u16, + pub host: String, +} + +pub enum Environment { + Local, + Production, +} + +impl Environment { + pub fn as_str(&self) -> &'static str { + match self { + Environment::Local => "local", + Environment::Production => "production", + } + } +} + +impl TryFrom for Environment { + type Error = String; + + fn try_from(s: String) -> Result { + match s.to_lowercase().as_str() { + "local" => Ok(Environment::Local), + "production" => Ok(Environment::Production), + e => Err(format!( + "{} is not a supported environment. Use `local` or `production`", + e + )), + } + } } pub fn get_configuration() -> Result { + let base_path = std::env::current_dir().expect("Failed to determine the current directory"); + let configuration_directory = base_path.join("configuration"); + + // Detect current environment, default to LOCAL + let environment: Environment = std::env::var("APP_ENVIRONMENT") + .unwrap_or_else(|_| "local".into()) + .try_into() + .expect("Failed to parse APP_ENVIRONMENT"); + let environment_filename = format!("{}.yaml", environment.as_str()); + // initialise config reader let settings = config::Config::builder() - .add_source(config::File::new( - "configuration.yaml", - config::FileFormat::Yaml, + .add_source(config::File::from( + configuration_directory.join("base.yaml"), )) + .add_source(config::File::from( + configuration_directory.join(environment_filename), + )) + .add_source( + config::Environment::with_prefix("APP") + .prefix_separator("_") + .separator("__"), + ) .build()?; + settings.try_deserialize::() } impl DatabaseSettings { - pub fn connection_string(&self) -> Secret { - Secret::new(format!( - "postgres://{}:{}@{}:{}/{}", - self.username, - self.password.expose_secret(), - self.host, - self.port, - self.database_name - )) + pub fn without_db(&self) -> PgConnectOptions { + let ssl_mode = if self.require_ssl { + PgSslMode::Require + } else { + PgSslMode::Prefer + }; + PgConnectOptions::new() + .username(&self.username) + .password(self.password.expose_secret()) + .host(&self.host) + .port(self.port) + .ssl_mode(ssl_mode) } - pub fn connection_string_without_db(&self) -> Secret { - Secret::new(format!( - "postgres://{}:{}@{}:{}", - self.username, - self.password.expose_secret(), - self.host, - self.port - )) + pub fn with_db(&self) -> PgConnectOptions { + self.without_db().database(&self.database_name) } } diff --git a/src/main.rs b/src/main.rs index 7594a49..d5e05f1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,7 @@ use mail_app::configuration::get_configuration; use mail_app::startup::run; use mail_app::telemetry::{get_subscriber, init_subscriber}; -use secrecy::ExposeSecret; -use sqlx::postgres::PgPool; +use sqlx::postgres::PgPoolOptions; use std::net::TcpListener; #[tokio::main] @@ -11,11 +10,11 @@ async fn main() -> Result<(), std::io::Error> { init_subscriber(subscriber); let configuration = get_configuration().expect("Failed to read configuration"); - let connection_pool = - PgPool::connect(&configuration.database.connection_string().expose_secret()) - .await - .expect("Failed to connect to Postgres."); - let address = format!("127.0.0.1:{}", configuration.application_port); + let connection_pool = PgPoolOptions::new().connect_lazy_with(configuration.database.with_db()); + let address = format!( + "{}:{}", + configuration.application.host, configuration.application.port + ); let listener = TcpListener::bind(address)?; run(listener, connection_pool)?.await } diff --git a/tests/health_check.rs b/tests/health_check.rs index 21405fc..8277a07 100644 --- a/tests/health_check.rs +++ b/tests/health_check.rs @@ -2,8 +2,7 @@ use mail_app::configuration::{get_configuration, DatabaseSettings}; use mail_app::startup::run; use mail_app::telemetry::{get_subscriber, init_subscriber}; use once_cell::sync::Lazy; -use secrecy::ExposeSecret; -use sqlx::{Executor, PgPool}; +use sqlx::{Connection, Executor, PgConnection, PgPool}; use std::net::TcpListener; use uuid::Uuid; @@ -47,7 +46,7 @@ async fn spawn_app() -> TestApp { pub async fn configure_database(config: &DatabaseSettings) -> PgPool { // Create Database - let connection = PgPool::connect(&config.connection_string_without_db().expose_secret()) + let mut connection = PgConnection::connect_with(&config.without_db()) .await .expect("Failed to connect to Postgres."); connection @@ -56,7 +55,7 @@ pub async fn configure_database(config: &DatabaseSettings) -> PgPool { .expect("Failed to create database."); // Migrate Database - let connection_pool = PgPool::connect(&config.connection_string().expose_secret()) + let connection_pool = PgPool::connect_with(config.with_db()) .await .expect("Failed to connect to Postgres."); sqlx::migrate!("./migrations")