From 37c4f22fe2f9ca4f3d1081bd00bf9973a4ab51d4 Mon Sep 17 00:00:00 2001 From: KristofferMorsing Date: Tue, 7 Apr 2026 12:33:41 +0200 Subject: [PATCH] new .workflow --- .workflow/01_container_build.sh | 17 +-- .workflow/01_dotnet_build.sh | 27 +--- .workflow/DotDockerbuilder10.sh | 205 ----------------------------- .workflow/Jenkinsfile | 97 +------------- .workflow/README.txt | 15 --- .workflow/build_markdown_report.sh | 106 --------------- .workflow/create_gitea_issue.sh | 77 ----------- .workflow/evaluate_threshold.sh | 26 ---- .workflow/filter_text_files.sh | 169 ------------------------ .workflow/get_changed_files.sh | 33 ----- .workflow/scan_with_ollama.sh | 133 ------------------- .workflow/verify_environment.sh | 18 --- 12 files changed, 7 insertions(+), 916 deletions(-) delete mode 100644 .workflow/DotDockerbuilder10.sh delete mode 100644 .workflow/README.txt delete mode 100644 .workflow/build_markdown_report.sh delete mode 100644 .workflow/create_gitea_issue.sh delete mode 100644 .workflow/evaluate_threshold.sh delete mode 100644 .workflow/filter_text_files.sh delete mode 100644 .workflow/get_changed_files.sh delete mode 100644 .workflow/scan_with_ollama.sh delete mode 100644 .workflow/verify_environment.sh diff --git a/.workflow/01_container_build.sh b/.workflow/01_container_build.sh index 2b74a15..565433b 100644 --- a/.workflow/01_container_build.sh +++ b/.workflow/01_container_build.sh @@ -2,21 +2,6 @@ /bin/echo "$0 - Building all containers [$(pwd)]" echo "USER: $UID" -# Parse owner/repo from GIT_URL -: "${GIT_URL:?Missing GIT_URL in Jenkins env}" -# Handle formats: https://gitea.example.com/owner/repo.git or git@gitea.example.com:owner/repo.git -OWNER="" -REPO="" -if [[ "$GIT_URL" =~ [:\/]([^\/:]+)\/([^\/\.]+)(\.git)?$ ]]; then - OWNER="${BASH_REMATCH[1]}" - REPO="${BASH_REMATCH[2]}" -fi -if [[ -z "$OWNER" || -z "$REPO" ]]; then - echo "[ERROR] Could not determine owner/repo from GIT_URL=$GIT_URL" - exit 1 -fi -echo "Will publish package at [$OWNER]" - echo "Will build:" /bin/find ./ -type f -name "*.csproj" | while read line; do solution=$(echo "$line" | cut -f 2 -d '/') @@ -29,6 +14,6 @@ done solution=$(echo "$line" | cut -f 2 -d '/') project=$(echo "$line" | cut -f 3 -d '/') echo "Building [$solution] [$project]" - .workflow/DotDockerbuilder10.sh --solution $solution --project $project --gituser $OWNER + .workflow/DotDockerbuilder8.sh --solution $solution --project $project done diff --git a/.workflow/01_dotnet_build.sh b/.workflow/01_dotnet_build.sh index 13439e1..97c6de2 100644 --- a/.workflow/01_dotnet_build.sh +++ b/.workflow/01_dotnet_build.sh @@ -1,26 +1,5 @@ #!/bin/bash +/bin/echo "$0 - Building all projects" +echo "USER: $UID" +/bin/find ./ -type f -name "*.csproj" -exec dotnet build "{}" \; -echo "OK" > jenkins.result -# Find all .csproj files and loop through them -find . -type f -name "*.csproj" | while read -r csproj; do - echo "Building $csproj..." - - # Run dotnet build on the csproj file - dotnet build "$csproj" - - # If dotnet build fails (non-zero exit code), exit the script with an error code - if [ $? -ne 0 ]; then - echo "Build failed for $csproj. Exiting with error." - echo "ERROR" > jenkins.result - fi -done - -RESULT=$(cat jenkins.result) -echo "RESULT=$RESULT" -if [ "$RESULT" == "OK" ]; then - echo "All builds succeeded." - exit 0 -else - echo "Errors were recorded" - exit 1 -fi diff --git a/.workflow/DotDockerbuilder10.sh b/.workflow/DotDockerbuilder10.sh deleted file mode 100644 index 2e018d6..0000000 --- a/.workflow/DotDockerbuilder10.sh +++ /dev/null @@ -1,205 +0,0 @@ -#!/bin/bash - -VERSION="2025.05.12-1430" - -usage() { - cat << EOF -DotDockerBuilder8 Version:$VERSION - Arguments: - -D|--directory (optional) path to solution folder - -S|--solution (mandatory) solution name - -P|--project (mandatory) project name - -p|--port (mandatory) exposed port - -G|--git (optional) git URI (gitea.a.ucnit.eu) - -U|--gituser (optional) authorized git user (docker | gitea organization) - -A|--accesstoken (optional) accesstoken for git user - -h|--help this text -EOF - exit -} - -[ $# -eq 0 ] && usage - -DIRECTORY="" -SOLUTION="" -PROJECT="" -EXPOSED_PORT="8080" # Since 2024, port 8080 is the standard web port for .NET - -while [[ $# -gt 0 ]]; do - case $1 in - -D|--directory) - DIRECTORY="$2" - shift # past argument - shift # past value - ;; - -S|--solution) - SOLUTION="$2" - shift # past argument - shift # past value - ;; - -P|--project) - PROJECT="$2" - shift # past argument - shift # past value - ;; - -p|--port) - EXPOSED_PORT="$2" - shift # past argument - shift # past value - ;; - -G|--git) - Git="$2" - shift # past argument - shift # past value - ;; - -U|--gituser) - GitUser="$2" - shift # past argument - shift # past value - ;; - -A|--accesstoken) - AccessToken="$2" - shift # past argument - shift # past value - ;; - -h|--help) - usage - ;; - -*|--*) - echo "Unknown option $1" - usage - ;; - *) - shift # past argument - ;; - esac -done - -[ -n "$DIRECTORY" ] && cd $DIRECTORY - - - -if [ ! -e $SOLUTION/$PROJECT/$PROJECT.csproj ]; then - echo "Error, Not found: $SOLUTION/$PROJECT/$PROJECT.csproj" - echo "Executed from $(pwd)" - cat << EOF -Usage: - $0 (-S|--solution) solution (-P|--project) project (-p|--port) exposed_port - Please call from just outside solution folder -EOF - echo "Current directory: $(pwd)" - exit -fi - -RUNDIR=$pwd -thisArch=$(uname -m) -myRuntime="UNKNOWN" -[ "$thisArch" == "x86_64" ] && myRuntime="linux-x64" -[ "$thisArch" == "aarch64" ] && myRuntime="linux-arm64" - -# Dockerfile.dotbuild is executed from parent of solution -cat << EOF > Dockerfile.dotbuild -# Version 2024.01.26 1702 -FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base -WORKDIR /app -EXPOSE $EXPOSED_PORT - -FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build -WORKDIR /src -COPY $SOLUTION ./ -RUN pwd -RUN dotnet restore "$PROJECT/$PROJECT.csproj" -RUN dotnet build "$PROJECT/$PROJECT.csproj" -c Release --self-contained --runtime $myRuntime -o /app/build -RUN ls -lR /app - -FROM build AS publish -RUN dotnet publish "$PROJECT/$PROJECT.csproj" -c Release --self-contained --runtime $myRuntime -o /app/publish -RUN ls -lR /app - -FROM base AS final -WORKDIR /app -COPY --from=publish /app/publish . -#ENTRYPOINT ["dotnet", "$PROJECT.dll"] -### New section -### CHECK_PORTS="mysql:3306 rabbitMQ:5672" -COPY start.dotbuild ./kajestartup.sh -RUN chmod 755 kajestartup.sh -CMD ./kajestartup.sh -RUN pwd -RUN ls -l -RUN cat ./kajestartup.sh -EOF - -cat << EOF > start.dotbuild -#!/bin/bash -echo "Version 22.10.15 1446" -if [ -n "\$CHECK_PORTS" ]; then - echo "Must check ports before start" - ! which nc >/dev/null && apt-get update && apt-get -y install netcat - for nn in \$(echo "\$CHECK_PORTS" | tr ';' ' '); do - xHost=\$(echo "\$nn" | cut -f 1 -d ':') - xPort=\$(echo "\$nn" | cut -f 2 -d ':') - echo "Checking Host: \$xHost Port: \$xPort" - while ! nc -z -v \$xHost \$xPort; do - echo "Waiting for Host: \$xHost Port: \$xPort" - sleep 5 - done - echo "Checking Host: \$xHost Port: \$xPort ... OK" - done - echo "Checking ports done..." -fi -# Adding DOCKER_HOST will cause an entry in /etc/hosts for local access to host platform -if [ -n "\$DOCKER_HOST" ] && grep -qv "KAJE" /etc/hosts; then - echo "Adding DOCKER_HOST to /etc/hosts" - b0=\$(cat /proc/net/route | sed -n '2,2 p' | cut -b 21-22) - b1=\$(cat /proc/net/route | sed -n '2,2 p' | cut -b 19-20) - b2=\$(cat /proc/net/route | sed -n '2,2 p' | cut -b 17-18) - b3=\$(cat /proc/net/route | sed -n '2,2 p' | cut -b 15-16) - echo "\$((16#\$b0)).\$((16#\$b1)).\$((16#\$b2)).\$((16#\$b3)) \$DOCKER_HOST" >> /etc/hosts - echo "# KAJE WAS HERE" >> /etc/hosts - echo "Changed /etc/hosts:" - cat /etc/hosts -fi -echo "Starting: \$(date)" -dotnet $PROJECT.dll -[ "\$ZOMBIE" == "YES" ] && echo "ZOMBIE STATE ENTERED" && sleep 1d -EOF - -# We are just outside solution folder -tagName=$(echo "$PROJECT" | tr '[:upper:]' '[:lower:]') - -pwd -if [ -e $SOLUTION/$PROJECT/$PROJECT.csproj ]; then - echo "Project file OK" -else - echo "Project file NOK" - exit -fi -echo "--------------" -cat ./Dockerfile.dotbuild -echo "--------------" - -echo "docker build --file ./Dockerfile.dotbuild --tag=$tagName:latest ." -sudo docker build --file ./Dockerfile.dotbuild --tag=$tagName:latest . - -if [ -n "$Git" ]; then - CONTAINERNAME=$(echo -n "$Git/$GitUser/$tagName" | tr '[:upper:]' '[:lower:]') - sudo docker image rm $CONTAINERNAME:latest - sudo docker image rm $CONTAINERNAME:$(date +%y%m%d) - sudo docker image tag $tagName:latest $CONTAINERNAME:latest - sudo docker image tag $tagName:latest $CONTAINERNAME:$(date +%y%m%d) - sudo docker image ls - echo "Pushing images to repository" - sudo docker login $Git --username=$GitUser --password=$AccessToken - echo "docker push $CONTAINERNAME:latest" - sudo docker push $CONTAINERNAME:latest - echo "docker push $CONTAINERNAME:$(date +%y%m%d)" - sudo docker push $CONTAINERNAME:$(date +%y%m%d) - - echo "Cleaning up: [$tagName:latest] [$CONTAINERNAME:latest] [$CONTAINERNAME:$(date +%y%m%d)]" - sudo docker image rm $tagName:latest - sudo docker image rm $CONTAINERNAME:latest $CONTAINERNAME:$(date +%y%m%d) -fi - -sudo docker image prune --force -echo "Done..." diff --git a/.workflow/Jenkinsfile b/.workflow/Jenkinsfile index 462d364..629a0f8 100644 --- a/.workflow/Jenkinsfile +++ b/.workflow/Jenkinsfile @@ -1,39 +1,7 @@ pipeline { agent { - label 'cicd-builder' + label 'cicd-builder' } - - environment { - // Provided by user - //OLLAMA_API_URI = "https://agent:12tf56so@ollama-api.a.ucnit.eu/v1" - //OLLAMA_API_KEY = credentials("b8d5c306-99e5-42a6-9825-4da275e9df89") - //GITEA_TOKEN = credentials('48bf7c4c-b4c8-4652-bc90-17440e959bf4') - //GITEA_API_URL = "https://gitea.a.ucnit.eu/api/v1" - - // Internal paths/artifacts - CHANGED_FILES_FILE = "changed_files.txt" - TEXT_FILES_FILE = "text_files.txt" - SCAN_JSON_FILE = "scan_results.json" - REPORT_FILE = "report.md" - - // Optional: Extend/override allowed text extensions (comma-separated, lowercase) - // TEXT_FILE_EXTS = "py,js,ts,java,cs,go,rb,php,rs,kt,scala,sh,bash,zsh,ps1,tf,tfvars,yaml,yml,json,ini,conf,properties,toml,xml,gradle,m,mm,sql,md,rst,txt" - } - - options { - timeout(time: 30, unit: 'MINUTES') - timestamps() - disableConcurrentBuilds() - buildDiscarder(logRotator(numToKeepStr: '20')) - } - - parameters { - string(name: 'OLLAMA_MODEL', defaultValue: 'llama3.1:8b', description: 'Ollama model to use (e.g., llama3.1, codellama, qwen2.5-coder)') - booleanParam(name: 'FAIL_ON_MEDIUM_OR_HIGH', defaultValue: true, description: 'Fail the build if any Medium or higher finding is present') - string(name: 'MAX_FILE_BYTES', defaultValue: '200000', description: 'Limit of bytes per file passed to model (to avoid context overflows)') - string(name: 'REQUEST_TIMEOUT_SEC', defaultValue: '180', description: 'Per-file analysis timeout (seconds)') - } - stages { stage('List folder content') { steps { @@ -42,73 +10,19 @@ pipeline { sh "export" } } - stage('Change to unix files') { steps { echo "Change to unix files" - sh "/bin/dos2unix .workflow/* .workflow/*" - sh "/bin/chmod +x .workflow/* .workflow/*" + sh "/bin/dos2unix .workflow/*" + sh "/bin/chmod 755 .workflow/*" } } - - stage('Prepare & Verify') { - steps { - sh 'chmod +x .workflow/*.sh || true' - sh '.workflow/verify_environment.sh' - } - } - stage('Build all projects') { steps { echo "Build all projects" sh ".workflow/01_dotnet_build.sh" } } - - stage('Detect Changes') { - steps { - sh ".workflow/get_changed_files.sh \"$CHANGED_FILES_FILE\"" - archiveArtifacts artifacts: "$CHANGED_FILES_FILE", allowEmptyArchive: true - } - } - - stage('Filter Text Files') { - steps { - sh ".workflow/filter_text_files.sh \"$CHANGED_FILES_FILE\" \"$TEXT_FILES_FILE\"" - archiveArtifacts artifacts: "$TEXT_FILES_FILE", allowEmptyArchive: true - } - } - - stage('Scan with Ollama') { - when { expression { fileExists('text_files.txt') && sh(script: "test -s text_files.txt", returnStatus: true) == 0 } } - steps { - sh ".workflow/scan_with_ollama.sh \"$TEXT_FILES_FILE\" \"$SCAN_JSON_FILE\" \"$OLLAMA_MODEL\" \"$MAX_FILE_BYTES\" \"$REQUEST_TIMEOUT_SEC\"" - archiveArtifacts artifacts: "$SCAN_JSON_FILE", allowEmptyArchive: true - } - } - - stage('Build Markdown Report') { - when { expression { fileExists('text_files.txt') && sh(script: "test -s text_files.txt", returnStatus: true) == 0 } } - steps { - sh ".workflow/build_markdown_report.sh \"$SCAN_JSON_FILE\" \"$REPORT_FILE\"" - archiveArtifacts artifacts: "$REPORT_FILE", allowEmptyArchive: true - } - } - - stage('Report to Gitea (Issue)') { - when { expression { fileExists('text_files.txt') && sh(script: "test -s text_files.txt", returnStatus: true) == 0 } } - steps { - sh ".workflow/create_gitea_issue.sh \"$REPORT_FILE\"" - } - } - - stage('Quality Gate') { - when { expression { fileExists('text_files.txt') && sh(script: "test -s text_files.txt", returnStatus: true) == 0 } } - steps { - sh ".workflow/evaluate_threshold.sh \"$SCAN_JSON_FILE\" \"$FAIL_ON_MEDIUM_OR_HIGH\"" - } - } - stage('Docker container release') { when { branch 'Docker' @@ -119,9 +33,4 @@ pipeline { } } } - post { - always { - archiveArtifacts allowEmptyArchive: true, artifacts: "${CHANGED_FILES_FILE}, ${TEXT_FILES_FILE}, ${SCAN_JSON_FILE}, ${REPORT_FILE}" - } - } } diff --git a/.workflow/README.txt b/.workflow/README.txt deleted file mode 100644 index d613629..0000000 --- a/.workflow/README.txt +++ /dev/null @@ -1,15 +0,0 @@ -Jenkinsfile with AI provisions - -The Build Agent should provide the following environment (in alphabetical order): -- GITEA_AGENT in clear text -- GITEA_API_URL in clear text -- GITEA_API_KEY in clear text -- GIT_USER in clear text -- MAX_FILE_BYTES in clear text -- OLLAMA_API_KEY through credentials:secret text -- OLLAMA_API_URI in clear text -- OLLAMA_MODEL in clear text -- REQUEST_TIMEOUT_SEC through credentials:secret text -- FILE_EXTS in clear text (cshtml,cs,js,json) - - diff --git a/.workflow/build_markdown_report.sh b/.workflow/build_markdown_report.sh deleted file mode 100644 index 3bc2536..0000000 --- a/.workflow/build_markdown_report.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -set -euo pipefail - -IN_JSON="${1:-scan_results.json}" -OUT_MD="${2:-report.md}" - -if [[ ! -s "$IN_JSON" ]]; then - echo "# Security Scan Report" > "$OUT_MD" - echo "_No results available (empty input)._" >> "$OUT_MD" - exit 0 -fi - -BRANCH="${BRANCH_NAME:-${GIT_BRANCH:-unknown}}" -SHA="${GIT_COMMIT:-unknown}" -SHORT_SHA="${SHA:0:7}" - -TOTAL_FILES=$(jq 'length' "$IN_JSON") -TOTAL_FINDINGS=$(jq '[.[].findings] | flatten | length' "$IN_JSON") -LOW=$(jq '[.[].findings] | flatten | map(select(.severity=="LOW")) | length' "$IN_JSON") -MED=$(jq '[.[].findings] | flatten | map(select(.severity=="MEDIUM")) | length' "$IN_JSON") -HIGH=$(jq '[.[].findings] | flatten | map(select(.severity=="HIGH")) | length' "$IN_JSON") -CRIT=$(jq '[.[].findings] | flatten | map(select(.severity=="CRITICAL")) | length' "$IN_JSON") - -{ - echo "# Security Scan Report (Ollama)" - echo "" - echo "- **Branch**: \`${BRANCH}\`" - echo "- **Commit**: \`${SHORT_SHA}\`" - echo "- **Files analyzed**: ${TOTAL_FILES}" - echo "- **Total findings**: ${TOTAL_FINDINGS}" - echo "" - echo "### Severity Summary" - echo "" - echo "| Severity | Count |" - echo "|-----------|------:|" - echo "| CRITICAL | ${CRIT} |" - echo "| HIGH | ${HIGH} |" - echo "| MEDIUM | ${MED} |" - echo "| LOW | ${LOW} |" - echo "" - echo "---" - echo "## Per-File Findings" - echo "" -} > "$OUT_MD" - -file_count=$(jq 'length' "$IN_JSON") -if [[ "$file_count" -eq 0 ]]; then - echo "_No files were scanned._" >> "$OUT_MD" - exit 0 -fi - -# Helper: safe cell formatter via jq (strip newlines and excessive whitespace) -# We do not export a function; we let jq do formatting inline per row. - -for i in $(seq 0 $((file_count-1))); do - file_path=$(jq -r ".[$i].file" "$IN_JSON") - findings_len=$(jq -r ".[$i].findings | length" "$IN_JSON") - - echo "### \`$file_path\`" >> "$OUT_MD" - echo "" >> "$OUT_MD" - - if [[ "$findings_len" -eq 0 ]]; then - echo "_No issues found._" >> "$OUT_MD" - echo "" >> "$OUT_MD" - continue - fi - - echo "| Severity | Line | Rule ID | Description | Recommendation |" >> "$OUT_MD" - echo "|----------|-----:|---------|-------------|----------------|" >> "$OUT_MD" - - # Notes: - # - Replace CRLF and LF with spaces via gsub("\r?\n"; " ") - # - Collapse all whitespace via gsub("[[:space:]]+"; " ") - # - Trim leading/trailing spaces using sub and gsub - jq -r ".[$i].findings[] | - [ - (.severity // \"\"), - (if (.line|type==\"number\") then (.line|tostring) else \"\" end), - (.rule_id // \"\"), - ((.description // \"\") - | gsub(\"\\r?\\n\"; \" \") - | gsub(\"[[:space:]]+\"; \" \") - | sub(\"^ \"; \"\") - | sub(\" $\"; \"\")), - ((.recommendation // \"\") - | gsub(\"\\r?\\n\"; \" \") - | gsub(\"[[:space:]]+\"; \" \") - | sub(\"^ \"; \"\") - | sub(\" $\"; \"\")) - ] - | \"| \" + (.[0]) + \" | \" + (.[1]) + \" | \" + (.[2]) + \" | \" + (.[3]) + \" | \" + (.[4]) + \" |\" - " "$IN_JSON" >> "$OUT_MD" - - # References (if present) - refs_count=$(jq ".[$i].findings | map(select(.references != null and (.references|length>0))) | length" "$IN_JSON") - if [[ "$refs_count" -gt 0 ]]; then - echo "" >> "$OUT_MD" - echo "**References**:" >> "$OUT_MD" - jq -r ".[$i].findings[] | select(.references != null) | .references[]? | \"- \" + tostring" "$IN_JSON" >> "$OUT_MD" - fi - - echo "" >> "$OUT_MD" -done - -echo "[INFO] Markdown report generated: $OUT_MD" -`` \ No newline at end of file diff --git a/.workflow/create_gitea_issue.sh b/.workflow/create_gitea_issue.sh deleted file mode 100644 index 23ae9e4..0000000 --- a/.workflow/create_gitea_issue.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -set -euo pipefail - -BODY_FILE="${1:-report.md}" - -if [[ ! -f "$BODY_FILE" ]]; then - echo "[WARN] Report file not found: $BODY_FILE. Skipping Gitea issue creation." - exit 0 -fi - -# If report says no files or no findings, you may decide NOT to create an issue. -#HAS_FINDINGS=$(grep -E "Total findings: [1-9][0-9]*" -i "$BODY_FILE" || true) -HAS_FINDINGS=$(cat report.md | grep "Total findings\*\*" | grep -v ": 0" || true) -if [[ -z "$HAS_FINDINGS" ]]; then - echo "[INFO] No findings detected. Not creating a Gitea issue." - exit 0 -fi - -: "${GITEA_API_URL:?Missing GITEA_API_URL}" -: "${GITEA_API_KEY:?Missing GITEA_API_KEY}" -: "${GITEA_AGENT:?Missing GITEA_AGENT}" -# GITEA_REPO_OWNER=$(git remote show origin | grep "^ Fetch" | cut -f 4 -d '/') -# echo "Will report issue to: [$GITEA_REPO_OWNER]" - -# Parse owner/repo from GIT_URL -: "${GIT_URL:?Missing GIT_URL in Jenkins env}" -# Handle formats: https://gitea.example.com/owner/repo.git or git@gitea.example.com:owner/repo.git -OWNER="" -REPO="" -if [[ "$GIT_URL" =~ [:\/]([^\/:]+)\/([^\/\.]+)(\.git)?$ ]]; then - OWNER="${BASH_REMATCH[1]}" - REPO="${BASH_REMATCH[2]}" -fi -if [[ -z "$OWNER" || -z "$REPO" ]]; then - echo "[ERROR] Could not determine owner/repo from GIT_URL=$GIT_URL" - exit 1 -fi -echo "Will report issue to [$OWNER][$REPO]" - -BRANCH="${BRANCH_NAME:-${GIT_BRANCH:-unknown}}" -SHA="${GIT_COMMIT:-unknown}" -SHORT_SHA="${SHA:0:7}" -JOB_URL="${BUILD_URL:-}" - -TITLE="[security-scan] ${OWNER}/${REPO} @ ${BRANCH} (${SHORT_SHA})" -LABELS='["security-scan","ollama"]' - -# Build JSON payload safely -BODY_JSON=$(jq -Rs . < "$BODY_FILE") -ASSIGNEES_JSON=$(jq -n --arg u "$GITEA_AGENT" 'if ($u|length) > 0 then [$u] else [] end') - -PAYLOAD=$(jq -n --arg title "$TITLE" --argjson body "$BODY_JSON" --argjson labels "$LABELS" --argjson assignees "$ASSIGNEES_JSON" \ - '{title:$title, body:$body, labels:$labels, assignees:$assignees}') - -PAYLOAD=$(jq -n --arg title "$TITLE" --argjson body "$BODY_JSON" --argjson labels "$LABELS" --argjson assignees "$ASSIGNEES_JSON" \ - '{title:$title, body:$body}') - -#DEBUG -API_URL="${GITEA_API_URL%/}/repos/${OWNER}/${REPO}/issues" -echo "[INFO] Creating Gitea issue at ${API_URL} [$GITEA_API_KEY]" -echo "[DEBUG] PAYLOAD=[$PAYLOAD]" - -# Gitea supports "token" auth header -RESP=$(curl -sS -X POST "$API_URL" \ - -H "Authorization: token $GITEA_API_KEY" \ - -H "Content-Type: application/json" \ - -d "$PAYLOAD") - -ISSUE_NUM=$(echo "$RESP" | jq -r '.number // empty') - -if [[ -n "$ISSUE_NUM" ]]; then - echo "[OK] Created issue #$ISSUE_NUM" -else - echo "[ERROR] Failed to create issue. Response:" - echo "$RESP" - exit 1 -fi \ No newline at end of file diff --git a/.workflow/evaluate_threshold.sh b/.workflow/evaluate_threshold.sh deleted file mode 100644 index f464850..0000000 --- a/.workflow/evaluate_threshold.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -euo pipefail - -JSON_FILE="${1:-scan_results.json}" -FAIL_ON_MED_OR_HIGH="${2:-true}" - -if [[ ! -s "$JSON_FILE" ]]; then - echo "[INFO] No scan results file present; nothing to evaluate." - exit 0 -fi - -MED_PLUS=$(jq '[.[].findings] | flatten | map(select(.severity=="MEDIUM" or .severity=="HIGH" or .severity=="CRITICAL")) | length' "$JSON_FILE") -CRIT=$(jq '[.[].findings] | flatten | map(select(.severity=="CRITICAL")) | length' "$JSON_FILE") -HIGH=$(jq '[.[].findings] | flatten | map(select(.severity=="HIGH")) | length' "$JSON_FILE") -MED=$(jq '[.[].findings] | flatten | map(select(.severity=="MEDIUM")) | length' "$JSON_FILE") - -echo "[INFO] Findings by severity => CRITICAL: ${CRIT}, HIGH: ${HIGH}, MEDIUM: ${MED}" - -if [[ "${FAIL_ON_MED_OR_HIGH,,}" == "true" ]]; then - if [[ "$MED_PLUS" -gt 0 ]]; then - echo "[FAIL] Medium or higher findings detected (${MED_PLUS}). Failing the build as configured." - exit 1 - fi -fi - -echo "[OK] Quality gate passed." \ No newline at end of file diff --git a/.workflow/filter_text_files.sh b/.workflow/filter_text_files.sh deleted file mode 100644 index 9774745..0000000 --- a/.workflow/filter_text_files.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# ====================================================================== -# Arguments -# ====================================================================== - -CHANGED_FILES="${1:-changed_files.txt}" -OUTPUT_FILE="${2:-text_files.txt}" -FILE_EXTS="${3:-}" # override whitelist extensions -CLI_EXCLUDES="${4:-}" # override exclusion paths -CLI_MAX_SIZE="${5:-}" # override max size per file (bytes) - -# ====================================================================== -# Extension whitelist (default) -# ====================================================================== - -DEFAULT_EXTS="c,cpp,h,hpp,py,js,jsx,ts,tsx,java,cs,go,rb,php,rs,kt,scala,sh,bash,zsh,ps1,tf,tfvars,yaml,yml,json,ini,conf,properties,toml,xml,gradle,m,mm,sql,md,rst,txt" - -if [ -n "$FILE_EXTS" ]; then - EXTS_RAW="$FILE_EXTS" -elif [ -n "${TEXT_FILE_EXTS:-}" ]; then - EXTS_RAW="$TEXT_FILE_EXTS" -else - EXTS_RAW="$DEFAULT_EXTS" -fi - -IFS=',' read -ra EXT_WHITELIST <<< "$(echo "$EXTS_RAW" | tr '[:upper:]' '[:lower:]' | tr -d ' ')" - -# ====================================================================== -# Exclusions paths -# ====================================================================== - -if [[ -n "$CLI_EXCLUDES" ]]; then - EXCLUDES_RAW="$CLI_EXCLUDES" -elif [[ -n "${SCAN_EXCLUDE_PATHS:-}" ]]; then - EXCLUDES_RAW="$SCAN_EXCLUDE_PATHS" -else - EXCLUDES_RAW="" -fi - -IFS=',' read -ra EXCLUDE_PATHS <<< "$(echo "$EXCLUDES_RAW" | tr -d ' ')" - -# ====================================================================== -# Maximum file-size limit -# ====================================================================== - -if [[ -n "$CLI_MAX_SIZE" ]]; then - MAX_SIZE="$CLI_MAX_SIZE" -elif [[ -n "${MAX_SCAN_FILE_BYTES:-}" ]]; then - MAX_SIZE="$MAX_SCAN_FILE_BYTES" -else - MAX_SIZE="0" # 0 = no limit -fi - -# ====================================================================== -# Special no-extension text files -# ====================================================================== - -SPECIAL_NAMES=( - "dockerfile" - "makefile" - ".gitignore" - ".gitattributes" - ".editorconfig" -) - -echo "[INFO] Extension whitelist: ${EXT_WHITELIST[*]}" -echo "[INFO] Exclusion paths: ${EXCLUDE_PATHS[*]}" -echo "[INFO] Max allowed file size (bytes): ${MAX_SIZE}" -echo "[INFO] Special text files: ${SPECIAL_NAMES[*]}" - -> "$OUTPUT_FILE" - -# ====================================================================== -# Helper: Check if file should be excluded by path -# ====================================================================== -should_exclude_path() { - local path="$1" - for ex in "${EXCLUDE_PATHS[@]}"; do - [[ -z "$ex" ]] && continue - if [[ "$path" == "$ex"* ]]; then - return 0 # true -> excluded - fi - done - return 1 # false -> not excluded -} - -# ====================================================================== -# Helper: Check file size against MAX_SIZE -# ====================================================================== -is_too_large() { - local file="$1" - local size - size=$(wc -c < "$file" | tr -d ' ') - if [ $MAX_SIZE -gt 0 ] && [ $size -gt $MAX_SIZE ]; then - echo "[DEBUG] Excluded due to size ($size > $MAX_SIZE): $file" - return 0 - fi - return 1 -} - -# ====================================================================== -# Main loop -# ====================================================================== -while IFS= read -r FILEPATH; do - [[ -z "$FILEPATH" ]] && continue - - # Skip files that do not exist - [[ ! -f "$FILEPATH" ]] && continue - - # 1. Path exclusion - if should_exclude_path "$FILEPATH"; then - echo "[DEBUG] Excluded by path: $FILEPATH" - continue - fi - - BASENAME="$(basename "$FILEPATH")" - BASENAME_LC="$(echo "$BASENAME" | tr '[:upper:]' '[:lower:]')" - - # 2. Special file names - for special in "${SPECIAL_NAMES[@]}"; do - if [[ "$BASENAME_LC" == "$special" ]]; then - if is_too_large "$FILEPATH"; then continue; fi - echo "$FILEPATH" >> "$OUTPUT_FILE" - continue 2 - fi - done - - # 3. Extract extension - if [[ "$FILEPATH" == *.* ]]; then - EXT="${FILEPATH##*.}" - else - EXT="" - fi - - EXT_LC="$(echo "$EXT" | tr '[:upper:]' '[:lower:]')" - - # 4. If no extension: skip (specials already handled) - if [[ -z "$EXT_LC" ]]; then - continue - fi - - # 5. If not in whitelist: skip - in_whitelist=false - for allowed in "${EXT_WHITELIST[@]}"; do - if [[ "$EXT_LC" == "$allowed" ]]; then - in_whitelist=true - break - fi - done - - [[ "$in_whitelist" == false ]] && continue - - # 6. File size check - if is_too_large "$FILEPATH"; then - continue - fi - - # 7. Add to output - echo "$FILEPATH" >> "$OUTPUT_FILE" - -done < "$CHANGED_FILES" - -# ====================================================================== -# Summary -# ====================================================================== -COUNT=$(wc -l < "$OUTPUT_FILE" | tr -d ' ') -echo "[INFO] Filter complete → $COUNT files selected" \ No newline at end of file diff --git a/.workflow/get_changed_files.sh b/.workflow/get_changed_files.sh deleted file mode 100644 index f4b025e..0000000 --- a/.workflow/get_changed_files.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -set -euo pipefail - -OUT_FILE="${1:-changed_files.txt}" -touch "$OUT_FILE" - -# Fetch quietly; repo might be shallow in CI -git fetch --all --quiet || true - -if [[ -n "${CHANGE_TARGET:-}" ]]; then - # Multibranch PR build: compare against target branch - git fetch origin "${CHANGE_TARGET}" --quiet || true - BASE="origin/${CHANGE_TARGET}" - echo "[INFO] PR build detected, diffing against ${BASE}..." - git diff --name-only --diff-filter=ACMRT "${BASE}...HEAD" > "$OUT_FILE" || true -else - # Branch build: diff from previous successful commit (best effort), else last commit - if [[ -n "${GIT_PREVIOUS_SUCCESSFUL_COMMIT:-}" ]]; then - BASE="${GIT_PREVIOUS_SUCCESSFUL_COMMIT}" - echo "[INFO] Diffing against previous successful commit ${BASE}..." - git diff --name-only --diff-filter=ACMRT "${BASE}" "HEAD" > "$OUT_FILE" || true - else - echo "[INFO] No previous successful commit found; using last commit diff (HEAD~1..HEAD)..." - git diff --name-only --diff-filter=ACMRT "HEAD~1" "HEAD" > "$OUT_FILE" || true - fi -fi - -# Clean blank lines, ensure file exists -sed -i '/^\s*$/d' "$OUT_FILE" || true -touch "$OUT_FILE" - -COUNT=$(wc -l < "$OUT_FILE" | tr -d ' ') -echo "[INFO] Changed files: ${COUNT}" \ No newline at end of file diff --git a/.workflow/scan_with_ollama.sh b/.workflow/scan_with_ollama.sh deleted file mode 100644 index 878dbb7..0000000 --- a/.workflow/scan_with_ollama.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash -set -euo pipefail - -FILES_LIST="${1:-text_files.txt}" -OUT_JSON="${2:-scan_results.json}" -OLLAMA_MODEL="${3:-llama3.1:8b}" -MAX_FILE_BYTES="${4:-200000}" -REQUEST_TIMEOUT_SEC="${5:-180}" - -if [[ ! -s "$FILES_LIST" ]]; then - echo "[]" > "$OUT_JSON" - echo "[INFO] No text files to scan. Generated empty JSON: $OUT_JSON" - exit 0 -fi - -TMP_DIR="$(mktemp -d)" -trap 'rm -rf "$TMP_DIR"' EXIT - -SYS_PROMPT=$'You are a precise secure code scanning assistant. Analyze the provided file content and return findings as STRICT JSON only with this schema:\n{\n "findings": [\n {\n "severity": "LOW|MEDIUM|HIGH|CRITICAL",\n "rule_id": "string-identifier",\n "line": number|null,\n "description": "short human-readable description",\n "recommendation": "concise fix guidance",\n "references": ["optional references URLs or identifiers"]\n }\n ]\n}\nRules:\n- Output ONLY JSON. No markdown, no code fences, no commentary.\n- If no issues, output {\"findings\":[]}.\n- Prefer concrete, reproducible findings over theoretical. Highlight secrets, insecure configs, vulnerable patterns, risky dependencies (if visible), command injection, path traversal, deserialization, SSRF, XSS, SQLi, hardcoded credentials, bad crypto, weak TLS, etc.\n- Provide line numbers when confidently known; otherwise null.\n' - -SYS_PROMPT=$(cat << EOF -You are a precise secure code scanning assistant. Analyze the provided file content and return findings as STRICT JSON only with this schema: -{ - "findings": [ - { - "severity": "LOW|MEDIUM|HIGH|CRITICAL", - "rule_id": "string-identifier", - "line": number|null, - "description": "short human-readable description", - "recommendation": "concise fix guidance", - "references": ["optional references URLs or identifiers"] - } - ] -} -Rules: -- Output ONLY JSON. No markdown, no code fences, no commentary. -- If no issues, output {\"findings\":[]}. -- Prefer concrete, reproducible findings over theoretical. Highlight secrets, insecure configs, vulnerable patterns, risky dependencies (if visible), command injection, path traversal, deserialization, SSRF, XSS, SQLi, hardcoded credentials, bad crypto, weak TLS, etc. -- Provide line numbers when confidently known; otherwise null. -EOF -) - - -export idx=0 -for file in $(cat "$FILES_LIST"); do - if [[ ! -f "$file" ]]; then - echo "[WARN] Skipping missing file: $file" - continue - fi - idx=$((idx+1)) - - ext="${file##*.}" - base="$(basename "$file")" - code_tmp="$TMP_DIR/code_$idx.bin" - head -c "$MAX_FILE_BYTES" "$file" | LC_ALL=C tr -d '\0' > "$code_tmp" - bytes=$(wc -c < "$code_tmp" | tr -d ' ') - - req_json="$TMP_DIR/req_$idx.json" - jq -n \ - --arg model "$OLLAMA_MODEL" \ - --arg sys "$SYS_PROMPT" \ - --arg f "$file" \ - --arg ext "$ext" \ - --rawfile code "$code_tmp" \ - '{ - model: $model, - temperature: 0, - stream: false, - messages: [ - {role: "system", content: $sys}, - {role: "user", content: ("Analyze the following " + - (if $ext == "" then "text" else $ext end) + - " file named \"" + $f + "\" and return ONLY JSON as specified. File content between <> markers: -<> -" + -$code + " -<> -")} - ] - }' > "$req_json" - - echo "[INFO] [$idx] Scanning $file (${bytes} bytes) with model=$OLLAMA_MODEL ..." - resp_json="$TMP_DIR/resp_$idx.json" - set +e - http_code=$(curl -sS -w "%{http_code}" -o "$resp_json" \ - --max-time "$REQUEST_TIMEOUT_SEC" --connect-timeout 10 \ - -H "Authorization: Bearer ${OLLAMA_API_KEY}" \ - -H "Content-Type: application/json" \ - -d @"$req_json" \ - "${OLLAMA_API_URI%/}/chat/completions") - rc=$? - set -e - - if [[ $rc -ne 0 || "$http_code" -lt 200 || "$http_code" -ge 300 ]]; then - echo "[ERROR] Ollama request failed for $file (rc=$rc, http=$http_code)." - # Create a synthetic "tool-error" entry - echo "{\"file\":\"$file\",\"findings\":[{\"severity\":\"LOW\",\"rule_id\":\"tool-error\",\"line\":null,\"description\":\"Ollama request failed (rc=$rc, http=$http_code)\",\"recommendation\":\"Re-run scan or check Ollama endpoint/credentials\",\"references\":[]}],\"model\":\"$OLLAMA_MODEL\",\"bytes_analyzed\":$bytes}" \ - > "$TMP_DIR/file_$idx.json" - continue - fi - - # Extract assistant message content (model output) - content=$(jq -r '.choices[0].message.content // ""' "$resp_json") - - # Try to parse content as JSON; if it has code fences, strip them - parsed_ok=1 - echo "$content" | jq . > "$TMP_DIR/json_${idx}_raw.json" 2>/dev/null || parsed_ok=0 - if [[ $parsed_ok -eq 0 ]]; then - content_stripped="$(printf "%s" "$content" | sed -E 's/^```(json)?[[:space:]]*//; s/```$//')" - echo "$content_stripped" | jq . > "$TMP_DIR/json_${idx}_raw.json" 2>/dev/null || parsed_ok=0 - else - parsed_ok=1 - fi - - if [[ $parsed_ok -eq 0 ]]; then - echo "[WARN] Model did not return valid JSON for $file. Wrapping as tool-error." - echo "{\"file\":\"$file\",\"findings\":[{\"severity\":\"LOW\",\"rule_id\":\"invalid-json\",\"line\":null,\"description\":\"Model returned non-JSON content\",\"recommendation\":\"Adjust prompt or model; ensure strict JSON output\",\"references\":[]}],\"model\":\"$OLLAMA_MODEL\",\"bytes_analyzed\":$bytes}" \ - > "$TMP_DIR/file_$idx.json" - else - # Normalize to object with findings array - jq --arg file "$file" --arg model "$OLLAMA_MODEL" --argjson bytes "$bytes" ' - . as $orig - | (if (has("findings") and (.findings|type=="array")) then $orig else {"findings": []} end) - | . + {file:$file, model:$model, bytes_analyzed:$bytes} - ' "$TMP_DIR/json_${idx}_raw.json" > "$TMP_DIR/file_$idx.json" - fi -done - -# Combine all per-file results into a single JSON array -jq -s '.' "$TMP_DIR"/file_*.json > "$OUT_JSON" -TOTAL=$(jq 'length' "$OUT_JSON") -echo "[INFO] Completed scan for ${TOTAL} file(s). Output: $OUT_JSON" -`` \ No newline at end of file diff --git a/.workflow/verify_environment.sh b/.workflow/verify_environment.sh deleted file mode 100644 index caed9df..0000000 --- a/.workflow/verify_environment.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Basic preflight checks. -for bin in git curl jq; do - if ! command -v "$bin" >/dev/null 2>&1; then - echo "[ERROR] Required binary not found: $bin" - exit 1 - fi -done - -# Validate envs that must be present. -: "${OLLAMA_API_URI:?Missing env OLLAMA_API_URI}" -: "${OLLAMA_API_KEY:?Missing env OLLAMA_API_KEY}" -: "${GITEA_API_URL:?Missing env GITEA_API_URL}" -: "${GITEA_API_KEY:?Missing env GITEA_APP_KEY}" - -echo "[OK] Preflight checks passed (git/curl/jq available and required env vars set)." \ No newline at end of file