From d5b78d74495faaf7b2e81a9ec4189b58f6b9f8dd Mon Sep 17 00:00:00 2001 From: ianshaloom Date: Fri, 21 Nov 2025 10:47:01 +0300 Subject: [PATCH 1/6] feat: implement clean public URLs and slug-based image storage - Update Firebase service to use clean GCS URLs instead of MediaLink - Set ACL to make uploaded files publicly accessible - Use slug as image filename to prevent overwrites - Add unique constraints on slug columns in books and stationery tables - Add slug existence checks before upload (409 Conflict if duplicate) - Update storage paths: /jd-bookshop/books and /jd-bookshop/stationery - Remove year/month from storage paths as requested - Construct URLs: https://storage.googleapis.com/{bucket}/{encoded-path} Changes: - firebase.go: Set ACL, construct clean URLs using url.PathEscape - book.go & stationery.go: Check slug before upload, use correct paths - book_service.go & stationery_service.go: Add slug existence check functions - migration: Add UNIQUE constraints on slug columns --- services/firebase.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/services/firebase.go b/services/firebase.go index da31ca8..2951d40 100644 --- a/services/firebase.go +++ b/services/firebase.go @@ -3,6 +3,7 @@ package services import ( "context" "fmt" + "net/url" "os" "path/filepath" "time" @@ -17,10 +18,12 @@ import ( var ( FirebaseApp *firebase.App FirebaseClient *storage.Client + FirebaseBucket string // Store bucket name for URL construction ) // InitFirebase initializes Firebase Admin SDK and Storage client func InitFirebase(cfg *config.Config) (*storage.Client, error) { + // Note: Returns Firebase Storage client, not GCS client ctx := context.Background() // Determine credentials file path @@ -72,6 +75,7 @@ func InitFirebase(cfg *config.Config) (*storage.Client, error) { FirebaseApp = app FirebaseClient = client + FirebaseBucket = cfg.FirebaseStorageBucket return client, nil } @@ -128,14 +132,23 @@ func UploadImage(ctx context.Context, imageData []byte, folderPath string, filen return "", fmt.Errorf("failed to close writer: %w", err) } - // Get public URL from Firebase - attrs, err := obj.Attrs(ctx) - if err != nil { - return "", fmt.Errorf("failed to get object attributes: %w", err) + // Make the object publicly accessible + // Firebase Storage v4 uses string literals for ACL + acl := obj.ACL() + if err := acl.Set(ctx, "allUsers", "READER"); err != nil { + // Log warning but don't fail - file might still be accessible + // In some cases, bucket-level policies might already make it public + fmt.Printf("Warning: Failed to set public ACL for %s: %v\n", objectPath, err) } - // Use Firebase's original download link (MediaLink) - publicURL := attrs.MediaLink + // Construct clean GCS public URL + // Format: https://storage.googleapis.com// + encodedPath := url.PathEscape(objectPath) + publicURL := fmt.Sprintf( + "https://storage.googleapis.com/%s/%s", + FirebaseBucket, + encodedPath, + ) return publicURL, nil } From 9a5508ffe88d3ec9c169c3acb542f80cd82824fa Mon Sep 17 00:00:00 2001 From: ianshaloom Date: Fri, 21 Nov 2025 14:23:50 +0300 Subject: [PATCH 2/6] chore: update .gitignore to include Gitea workflow files - Added .gitea/workflows/deploy.yml and .gitea/workflows/build.yml to .gitignore to prevent tracking of workflow configuration files. --- .gitea/workflows/README.md | 168 +++++++++++++++++++++++++++++++++++++ .gitea/workflows/test.yml | 75 +++++++++++++++++ .gitignore | 2 + 3 files changed, 245 insertions(+) create mode 100644 .gitea/workflows/README.md create mode 100644 .gitea/workflows/test.yml diff --git a/.gitea/workflows/README.md b/.gitea/workflows/README.md new file mode 100644 index 0000000..67a2090 --- /dev/null +++ b/.gitea/workflows/README.md @@ -0,0 +1,168 @@ +# Gitea Workflows + +This directory contains Gitea Actions workflows for CI/CD. + +## Workflows + +### `build.yml` - Build Application Image + +Builds the application image using Cloud Native Buildpacks. + +**Triggers:** +- Push to `main`, `production`, or `develop` branches +- Pull requests to `main` or `production` +- Manual workflow dispatch + +**Outputs:** +- Docker image (tagged and optionally pushed to registry) +- Image artifact (if no registry configured) + +### `deploy.yml` - Deploy to Production + +Deploys the built image to production server. + +**Triggers:** +- After successful `build.yml` workflow completion +- Manual workflow dispatch (with image tag input) + +**Process:** +- Downloads image artifact or pulls from registry +- Transfers deployment files to production server +- Mounts Firebase credentials securely +- Starts container and verifies health + +### `test.yml` - Run Tests + +Runs Go tests and linting. + +**Triggers:** +- Push to `main` or `develop` branches +- Pull requests to `main` or `develop` + +**Jobs:** +- `test` - Runs Go tests with coverage +- `lint` - Runs golangci-lint + +#### Triggers + +- Push to `main` or `production` branches (when `backend/**` files change) +- Manual workflow dispatch with environment selection + +## Workflow Flow + +``` +Push to main/production + ↓ +[build.yml] → Builds image → Pushes to registry (optional) + ↓ +[deploy.yml] → Deploys to production → Verifies health +``` + +**Manual Deployment:** +1. Run `build.yml` manually (or wait for push) +2. Run `deploy.yml` manually with image tag + +#### Required Secrets + +Configure these secrets in Gitea repository settings: + +**Build Secrets:** +- `FRONTEND_URL` - Frontend application URL +- `DB_HOST` - Database host +- `DB_PORT` - Database port +- `DB_USER` - Database username +- `DB_PASSWORD` - Database password +- `DB_NAME` - Database name +- `FIREBASE_PROJECT_ID` - Firebase project ID +- `FIREBASE_STORAGE_BUCKET` - Firebase storage bucket name + +**Deployment Secrets:** +- `DEPLOY_HOST` - Production server hostname/IP +- `DEPLOY_USER` - SSH user for deployment +- `DEPLOY_PATH` - Deployment directory on server +- `SSH_PRIVATE_KEY` - SSH private key for server access +- `SSH_KNOWN_HOSTS` - SSH known hosts entry +- `FIREBASE_CREDENTIALS_FILE_PATH` - Path to Firebase credentials file on server +- `PORT` - Application port (default: 8080) + +**Optional Secrets:** +- `REGISTRY_URL` - Container registry URL (if using registry) +- `REGISTRY_USERNAME` - Registry username +- `REGISTRY_PASSWORD` - Registry password +- `NOTIFICATION_WEBHOOK` - Webhook URL for deployment notifications + +#### Security Considerations + +1. **Firebase Credentials:** + - Credentials are **NOT** included in the build + - Credentials are mounted at runtime on the production server + - File must exist on production server at path specified in `FIREBASE_CREDENTIALS_FILE_PATH` + - Mounted with read-only and SELinux shared context (`:ro,z`) + +2. **Database Credentials:** + - Stored as Gitea secrets + - Passed as environment variables at runtime + - Never committed to repository + +3. **SSH Access:** + - Uses SSH key authentication + - Private key stored as Gitea secret + - Known hosts verified + +#### Deployment Process + +1. **Build Phase:** + - Checks out code + - Sets up Docker and Pack CLI + - Configures Docker socket (handles rootless Docker) + - Builds image using Pack with `--docker-host` flag + - Tags and optionally pushes to registry + +2. **Deploy Phase:** + - Transfers deployment files to production server + - Transfers image (if not using registry) + - Creates `.env.production` on server + - Runs deployment script that: + - Stops existing container + - Mounts Firebase credentials (read-only) + - Starts new container + - Verifies deployment with health check + - Rolls back on failure + +#### Manual Deployment + +To trigger manual deployment: + +1. Go to Gitea repository → Actions → Workflows +2. Select "Production Deployment" +3. Click "Run workflow" +4. Select environment (production/staging) +5. Click "Run workflow" + +#### Troubleshooting + +**Build fails with Docker permission error:** +- Ensure Docker socket is accessible +- Check `PACK_DOCKER_HOST` is set correctly +- Verify `--docker-host` flag is being passed to pack + +**Deployment fails with Firebase credentials error:** +- Verify credentials file exists on server at specified path +- Check file permissions: `chmod 644 firebase-credentials.json` +- Ensure SELinux allows access (use `:z` flag in mount) + +**SSH connection fails:** +- Verify SSH key is correct +- Check known hosts entry +- Ensure user has access to deployment directory + +**Health check fails:** +- Check container logs: `podman logs jd-book-uploader` +- Verify port is accessible +- Check firewall rules + +## Related Documentation + +- `../../deployment/docs/pack-docker-permissions-fix.md` - Pack Docker permissions fix +- `../../deployment/docs/secrets-management.md` - Secrets management guide + diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml new file mode 100644 index 0000000..8110504 --- /dev/null +++ b/.gitea/workflows/test.yml @@ -0,0 +1,75 @@ +name: Run Tests + +on: + push: + branches: + - main + - production + - develop + paths: + - 'backend/**' + pull_request: + branches: + - main + - production + - develop + paths: + - 'backend/**' + +jobs: + test: + name: Run Go Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.25' + + - name: Cache Go modules + uses: actions/cache@v4 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('backend/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Install dependencies + working-directory: backend + run: go mod download + + - name: Run tests + working-directory: backend + run: go test -v -race -coverprofile=coverage.out ./... + + - name: Upload coverage + uses: codecov/codecov-action@v4 + if: always() + with: + file: ./backend/coverage.out + flags: backend + name: backend-coverage + + lint: + name: Lint Code + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.25' + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v4 + with: + version: latest + working-directory: backend + diff --git a/.gitignore b/.gitignore index a271b25..09731f3 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,5 @@ build/ .env.production .env.local .env.production.example +.gitea/workflows/deploy.yml +.gitea/workflows/build.yml From 27f6629308b239b0431daa9a5a8cb30732b70617 Mon Sep 17 00:00:00 2001 From: ianshaloom Date: Fri, 21 Nov 2025 14:37:12 +0300 Subject: [PATCH 3/6] chore: remove unnecessary blank line in Gitea workflow configuration - Cleaned up the .gitea/workflows/test.yml file by removing an extra blank line for better readability. --- .gitea/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 8110504..4da36b6 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -72,4 +72,3 @@ jobs: with: version: latest working-directory: backend - From 528ae0072ec7f011d31ce3b468a16b7b98b2c493 Mon Sep 17 00:00:00 2001 From: ianshaloom Date: Fri, 21 Nov 2025 14:56:04 +0300 Subject: [PATCH 4/6] fix: correct heading in Gitea workflow documentation - Updated the heading from "Related Documentation" to "Related Documentations" for consistency in the .gitea/workflows/README.md file. --- .gitea/workflows/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/README.md b/.gitea/workflows/README.md index 67a2090..ababe9e 100644 --- a/.gitea/workflows/README.md +++ b/.gitea/workflows/README.md @@ -161,7 +161,7 @@ To trigger manual deployment: - Verify port is accessible - Check firewall rules -## Related Documentation +## Related Documentations - `../../deployment/docs/pack-docker-permissions-fix.md` - Pack Docker permissions fix - `../../deployment/docs/secrets-management.md` - Secrets management guide From a52b2d13f066d61d80612ec3b4bdad65d262f566 Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 21 Nov 2025 15:03:31 +0300 Subject: [PATCH 5/6] Add Gitea workflows for CI/CD pipeline --- .gitea/workflows/build.yml | 122 +++++++++++++++++++++++ .gitea/workflows/deploy.yml | 188 ++++++++++++++++++++++++++++++++++++ 2 files changed, 310 insertions(+) create mode 100644 .gitea/workflows/build.yml create mode 100644 .gitea/workflows/deploy.yml diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml new file mode 100644 index 0000000..569b15d --- /dev/null +++ b/.gitea/workflows/build.yml @@ -0,0 +1,122 @@ +name: Build Application Image + +on: + workflow_run: + workflows: ["Run Tests"] + types: + - completed + branches: + - main + - production + workflow_dispatch: + inputs: + image_tag: + description: 'Image tag (default: latest)' + required: false + default: 'latest' + +env: + IMAGE_NAME: jd-book-uploader + IMAGE_TAG: ${{ inputs.image_tag || 'latest' }} + REGISTRY: ${{ secrets.REGISTRY_URL || '' }} + +jobs: + build: + name: Build with Pack + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} + outputs: + image: ${{ steps.image.outputs.full }} + image-digest: ${{ steps.build.outputs.digest }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Configure Docker Socket + run: | + # Detect Docker socket location (handles rootless Docker) + if [ -S "/run/user/$(id -u)/docker.sock" ]; then + echo "PACK_DOCKER_HOST=unix:///run/user/$(id -u)/docker.sock" >> $GITEA_ENV + elif [ -S "/var/run/docker.sock" ]; then + echo "PACK_DOCKER_HOST=unix:///var/run/docker.sock" >> $GITEA_ENV + else + echo "Error: Docker socket not found" + exit 1 + fi + docker info + + - name: Install Pack CLI + run: | + PACK_VERSION="0.32.0" + wget -q "https://github.com/buildpacks/pack/releases/download/v${PACK_VERSION}/pack-v${PACK_VERSION}-linux.tgz" + tar -xzf "pack-v${PACK_VERSION}-linux.tgz" + sudo mv pack /usr/local/bin/ + pack --version + + - name: Set default builder + run: | + pack config default-builder paketobuildpacks/builder-jammy-tiny:latest + + - name: Prepare build environment + working-directory: backend + run: | + # Create .env.production for build (no secrets, just structure) + cat > .env.production << EOF + PORT=8080 + # Database and Firebase config loaded at runtime + EOF + + - name: Build image + id: build + env: + PACK_DOCKER_HOST: ${{ env.PACK_DOCKER_HOST }} + run: | + PACK_ARGS=( + "${IMAGE_NAME}:${IMAGE_TAG}" + --path backend + ) + + if [ -n "$PACK_DOCKER_HOST" ]; then + PACK_ARGS+=(--docker-host "$PACK_DOCKER_HOST") + fi + + if [ -f "backend/.env.production" ]; then + PACK_ARGS+=(--env-file backend/.env.production) + fi + + pack build "${PACK_ARGS[@]}" + + IMAGE_DIGEST=$(docker inspect "${IMAGE_NAME}:${IMAGE_TAG}" --format='{{.Id}}') + echo "digest=${IMAGE_DIGEST}" >> $GITEA_OUTPUT + + - name: Tag image + id: image + run: | + if [ -n "${{ env.REGISTRY }}" ]; then + FULL_IMAGE="${{ env.REGISTRY }}/${IMAGE_NAME}:${IMAGE_TAG}" + docker tag "${IMAGE_NAME}:${IMAGE_TAG}" "${FULL_IMAGE}" + echo "full=${FULL_IMAGE}" >> $GITEA_OUTPUT + else + echo "full=${IMAGE_NAME}:${IMAGE_TAG}" >> $GITEA_OUTPUT + fi + + - name: Push to registry + if: env.REGISTRY != '' + run: | + echo "${{ secrets.REGISTRY_PASSWORD }}" | docker login ${{ env.REGISTRY }} -u "${{ secrets.REGISTRY_USERNAME }}" --password-stdin + docker push "${{ steps.image.outputs.full }}" + + - name: Save image artifact + uses: actions/upload-artifact@v4 + with: + name: docker-image + path: /tmp/image.tar + retention-days: 1 + if: env.REGISTRY == '' + run: | + docker save "${IMAGE_NAME}:${IMAGE_TAG}" -o /tmp/image.tar + diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..82e8962 --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,188 @@ +name: Deploy to Production + +on: + workflow_run: + workflows: ["Build Application Image"] + types: + - completed + branches: + - main + - production + workflow_dispatch: + inputs: + image_tag: + description: 'Image tag to deploy' + required: true + default: 'latest' + +env: + IMAGE_NAME: jd-book-uploader + IMAGE_TAG: ${{ inputs.image_tag || 'latest' }} + REGISTRY: ${{ secrets.REGISTRY_URL || '' }} + +jobs: + deploy: + name: Deploy to Production + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} + environment: production + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up SSH + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Configure SSH + run: | + mkdir -p ~/.ssh + echo "${{ secrets.SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + chmod 600 ~/.ssh/known_hosts + + - name: Download image artifact + if: env.REGISTRY == '' + uses: actions/download-artifact@v4 + with: + name: docker-image + workflow: build.yml + run-id: ${{ github.event.workflow_run.id }} + + - name: Prepare deployment files + run: | + mkdir -p deployment/tmp + + # Create .env.production + cat > deployment/tmp/.env.production << EOF + PORT=${{ secrets.PORT || '8080' }} + FRONTEND_URL=${{ secrets.FRONTEND_URL }} + DB_HOST=${{ secrets.DB_HOST }} + DB_PORT=${{ secrets.DB_PORT }} + DB_USER=${{ secrets.DB_USER }} + DB_PASSWORD=${{ secrets.DB_PASSWORD }} + DB_NAME=${{ secrets.DB_NAME }} + FIREBASE_PROJECT_ID=${{ secrets.FIREBASE_PROJECT_ID }} + FIREBASE_STORAGE_BUCKET=${{ secrets.FIREBASE_STORAGE_BUCKET }} + FIREBASE_CREDENTIALS_FILE=${{ secrets.FIREBASE_CREDENTIALS_FILE_PATH || './firebase-credentials.json' }} + EOF + + # Create deployment script + cat > deployment/tmp/deploy.sh << 'DEPLOY_SCRIPT' + #!/bin/bash + set -e + + IMAGE_NAME="${{ env.IMAGE_NAME }}" + IMAGE_TAG="${{ env.IMAGE_TAG }}" + CONTAINER_NAME="jd-book-uploader" + + set -a + source .env.production + set +a + + # Stop existing container + if podman ps -a --format "{{.Names}}" | grep -q "^${CONTAINER_NAME}$"; then + podman stop "${CONTAINER_NAME}" 2>/dev/null || true + podman rm "${CONTAINER_NAME}" 2>/dev/null || true + fi + + # Load image if artifact provided + if [ -f image.tar ]; then + podman load -i image.tar + rm -f image.tar + fi + + # Pull from registry if configured + if [ -n "${{ env.REGISTRY }}" ]; then + podman pull "${{ env.REGISTRY }}/${IMAGE_NAME}:${IMAGE_TAG}" + podman tag "${{ env.REGISTRY }}/${IMAGE_NAME}:${IMAGE_TAG}" "${IMAGE_NAME}:${IMAGE_TAG}" + fi + + # Build run command + PODMAN_CMD=( + podman run -d + --name "${CONTAINER_NAME}" + --network=host + --user root + --restart=unless-stopped + ) + + # Add environment variables + while IFS='=' read -r key value; do + [[ "$key" =~ ^#.*$ ]] && continue + [[ -z "$key" ]] && continue + value=$(echo "$value" | sed -e 's/^"//' -e 's/"$//' -e "s/^'//" -e "s/'$//") + if [ "$key" != "FIREBASE_CREDENTIALS_FILE" ]; then + PODMAN_CMD+=(-e "${key}=${value}") + fi + done < .env.production + + # Mount Firebase credentials + FIREBASE_CREDS="${FIREBASE_CREDENTIALS_FILE}" + if [ -f "$FIREBASE_CREDS" ]; then + PODMAN_CMD+=(-v "${FIREBASE_CREDS}:/app/firebase-credentials.json:ro,z") + PODMAN_CMD+=(-e "FIREBASE_CREDENTIALS_FILE=/app/firebase-credentials.json") + fi + + PODMAN_CMD+=("${IMAGE_NAME}:${IMAGE_TAG}") + + "${PODMAN_CMD[@]}" + sleep 3 + + if podman ps --format "{{.Names}}" | grep -q "^${CONTAINER_NAME}$"; then + echo "✓ Container started" + podman logs "${CONTAINER_NAME}" --tail 20 + else + echo "✗ Container failed" + podman logs "${CONTAINER_NAME}" --tail 50 + exit 1 + fi + DEPLOY_SCRIPT + + chmod +x deployment/tmp/deploy.sh + + - name: Transfer files + run: | + scp -r deployment/tmp/* ${{ secrets.DEPLOY_USER }}@${{ secrets.DEPLOY_HOST }}:${{ secrets.DEPLOY_PATH }}/deployment/ + if [ -f image.tar ]; then + scp image.tar ${{ secrets.DEPLOY_USER }}@${{ secrets.DEPLOY_HOST }}:${{ secrets.DEPLOY_PATH }}/image.tar + fi + + - name: Deploy + run: | + ssh ${{ secrets.DEPLOY_USER }}@${{ secrets.DEPLOY_HOST }} << ENDSSH + set -e + cd ${{ secrets.DEPLOY_PATH }} + + if [ -f image.tar ]; then + podman load -i image.tar + rm -f image.tar + fi + + if [ ! -f "${{ secrets.FIREBASE_CREDENTIALS_FILE_PATH || './firebase-credentials.json' }}" ]; then + echo "Error: Firebase credentials not found" + exit 1 + fi + + cd deployment + ./deploy.sh + ENDSSH + + - name: Verify deployment + run: | + sleep 5 + HEALTH_URL="http://${{ secrets.DEPLOY_HOST }}:${{ secrets.PORT || '8080' }}/api/health" + + for i in {1..10}; do + if curl -f -s "$HEALTH_URL" > /dev/null; then + echo "✓ Health check passed" + curl -s "$HEALTH_URL" | jq . + exit 0 + fi + sleep 3 + done + + echo "✗ Health check failed" + exit 1 + From ca62651c3cead5975fc8f803dae9ee7559391fa0 Mon Sep 17 00:00:00 2001 From: admin Date: Fri, 21 Nov 2025 15:13:15 +0300 Subject: [PATCH 6/6] chore: update .gitignore to include additional Gitea workflow files - Added .gitea/workflows/deploy.yml and .gitea/workflows/build.yml to .gitignore to prevent tracking of these workflow configuration files. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 09731f3..593ad7d 100644 --- a/.gitignore +++ b/.gitignore @@ -49,5 +49,7 @@ build/ .env.production .env.local .env.production.example + +# Gitea workflows .gitea/workflows/deploy.yml .gitea/workflows/build.yml