From 185cc03df7ba5b9a86aed07ed05143229b92785c Mon Sep 17 00:00:00 2001 From: saqib mir Date: Thu, 23 Jul 2026 11:31:26 +0530 Subject: [PATCH] docker issue --- CMakeLists.txt | 4 +- docker-compose.yml | 9 +- engine/CMakeLists.txt | 37 ++++---- engine/include/pdfengine/display_list.hpp | 10 +-- engine/include/pdfengine/graphics_state.hpp | 4 +- engine/tests/CMakeLists.txt | 4 + gateway/Dockerfile | 93 +++++++++++++++++++-- third_party/pdfium/build_pdfium.sh | 20 +++-- 8 files changed, 138 insertions(+), 43 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d483f9..1b99383 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,7 +73,9 @@ find_package(freetype CONFIG REQUIRED) find_package(harfbuzz CONFIG REQUIRED) find_package(spdlog CONFIG REQUIRED) find_package(nlohmann_json CONFIG REQUIRED) -find_package(qpdf CONFIG REQUIRED) +if(PDFENGINE_WITH_QPDF) + find_package(qpdf CONFIG REQUIRED) +endif() if(WIN32 AND DEFINED VCPKG_TARGET_TRIPLET) link_directories("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/lib") link_directories("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/debug/lib") diff --git a/docker-compose.yml b/docker-compose.yml index 5358d22..b759a91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,13 +1,14 @@ services: gateway: build: - context: ./gateway - dockerfile: Dockerfile + context: . + dockerfile: gateway/Dockerfile + network: host image: pdf-engine-gateway:dev container_name: pdf-engine-gateway environment: PDFENGINE_ENVIRONMENT: dev - PDFENGINE_ENGINE_AVAILABLE: "false" + PDFENGINE_ENGINE_AVAILABLE: "true" PORT: 8000 ports: - "8000:8000" @@ -29,7 +30,7 @@ services: image: pdf-engine-frontend:dev container_name: pdf-engine-frontend environment: - VITE_GATEWAY_URL: http://gateway:8000 + VITE_GATEWAY_URL: http://localhost:8000 ports: - "5173:5173" volumes: diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 3076e68..512b7de 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -5,32 +5,18 @@ configure_file( find_package(PNG REQUIRED) -add_library(pdfengine STATIC +add_library(pdfengine OBJECT src/core/engine_info.cpp src/core/graphics_state.cpp src/core/display_list.cpp src/core/path_interpreter.cpp src/core/skia_renderer.cpp - src/parser/pdfium_loader.cpp - src/parser/pdfium_document.cpp - src/parser/pdfium_internal.cpp - src/parser/pdfium_reflow.cpp - src/parser/pdfium_page.cpp - src/parser/pdfium_page_model.cpp - src/parser/pdfium_fonts.cpp - src/parser/pdfium_edit.cpp - src/parser/pdfium_edit_replace.cpp - src/parser/pdfium_edit_reflow.cpp - src/parser/pdfium_edit_annotations.cpp - src/parser/pdfium_edit_pages.cpp - src/parser/pdfium_edit_images.cpp src/parser/content_stream_parser.cpp src/parser/decoration_builder.cpp src/text/selection.cpp src/fonts/face/font_face.cpp src/fonts/face/free_type_manager.cpp src/fonts/loader/font_resolver.cpp - src/fonts/pdf_fonts/font_loader.cpp src/fonts/shaping/hb_shaper.cpp src/fonts/cache/glyph_bitmap.cpp src/fonts/cache/glyph_cache.cpp @@ -48,6 +34,7 @@ add_library(pdfengine STATIC src/fonts/pdf_fonts/encoding/cjk_collection_db.cpp ) add_library(pdfengine::pdfengine ALIAS pdfengine) +set_target_properties(pdfengine PROPERTIES POSITION_INDEPENDENT_CODE ON) target_include_directories(pdfengine PUBLIC @@ -69,8 +56,26 @@ target_link_libraries(pdfengine ) if(PDFENGINE_WITH_PDFIUM) + target_sources(pdfengine PRIVATE + src/parser/pdfium_loader.cpp + src/parser/pdfium_document.cpp + src/parser/pdfium_internal.cpp + src/parser/pdfium_reflow.cpp + src/parser/pdfium_page.cpp + src/parser/pdfium_page_model.cpp + src/parser/pdfium_fonts.cpp + src/parser/pdfium_edit.cpp + src/parser/pdfium_edit_replace.cpp + src/parser/pdfium_edit_reflow.cpp + src/parser/pdfium_edit_annotations.cpp + src/parser/pdfium_edit_pages.cpp + src/parser/pdfium_edit_images.cpp + ) target_link_libraries(pdfengine PRIVATE pdfium::pdfium) - target_compile_definitions(pdfengine PRIVATE PDFENGINE_WITH_PDFIUM) + target_compile_definitions(pdfengine PUBLIC PDFENGINE_WITH_PDFIUM) + # PDFium statically bundles its own libjpeg, zlib, etc. which conflicts with vcpkg. + # We use LLD, so we can safely allow multiple definitions to pick the first one. + target_link_options(pdfengine PUBLIC "-Wl,--allow-multiple-definition") endif() if(EMSCRIPTEN) diff --git a/engine/include/pdfengine/display_list.hpp b/engine/include/pdfengine/display_list.hpp index cd98c30..801bed2 100644 --- a/engine/include/pdfengine/display_list.hpp +++ b/engine/include/pdfengine/display_list.hpp @@ -32,34 +32,34 @@ struct SetTransformCommand : public Command { struct FillRectCommand : public Command { float x, y, width, height; - FillRectCommand(float x, float y, float w, float h) : x(x), y(y), width(w), height(h) {} + FillRectCommand(float _x, float _y, float w, float h) : x(_x), y(_y), width(w), height(h) {} void accept(CommandVisitor& visitor) const override; }; struct DrawTextCommand : public Command { std::string text; float x, y; - DrawTextCommand(std::string text, float x, float y) : text(std::move(text)), x(x), y(y) {} + DrawTextCommand(std::string _text, float _x, float _y) : text(std::move(_text)), x(_x), y(_y) {} void accept(CommandVisitor& visitor) const override; }; struct FillPathCommand : public Command { Path path; FillRule rule; - explicit FillPathCommand(Path path, FillRule rule = FillRule::NonZero) : path(std::move(path)), rule(rule) {} + explicit FillPathCommand(Path _path, FillRule _rule = FillRule::NonZero) : path(std::move(_path)), rule(_rule) {} void accept(CommandVisitor& visitor) const override; }; struct StrokePathCommand : public Command { Path path; - explicit StrokePathCommand(Path path) : path(std::move(path)) {} + explicit StrokePathCommand(Path _path) : path(std::move(_path)) {} void accept(CommandVisitor& visitor) const override; }; struct FillStrokePathCommand : public Command { Path path; FillRule rule; - explicit FillStrokePathCommand(Path path, FillRule rule = FillRule::NonZero) : path(std::move(path)), rule(rule) {} + explicit FillStrokePathCommand(Path _path, FillRule _rule = FillRule::NonZero) : path(std::move(_path)), rule(_rule) {} void accept(CommandVisitor& visitor) const override; }; diff --git a/engine/include/pdfengine/graphics_state.hpp b/engine/include/pdfengine/graphics_state.hpp index 36c59ed..9542bb2 100644 --- a/engine/include/pdfengine/graphics_state.hpp +++ b/engine/include/pdfengine/graphics_state.hpp @@ -11,8 +11,8 @@ struct Matrix { float e = 0.0f, f = 0.0f; Matrix() = default; - Matrix(float a, float b, float c, float d, float e, float f) - : a(a), b(b), c(c), d(d), e(e), f(f) {} + Matrix(float _a, float _b, float _c, float _d, float _e, float _f) + : a(_a), b(_b), c(_c), d(_d), e(_e), f(_f) {} [[nodiscard]] Matrix multiply(const Matrix& other) const noexcept; diff --git a/engine/tests/CMakeLists.txt b/engine/tests/CMakeLists.txt index 2970ed4..e1b053d 100644 --- a/engine/tests/CMakeLists.txt +++ b/engine/tests/CMakeLists.txt @@ -38,6 +38,10 @@ if(PDFENGINE_WITH_SKIA) target_link_libraries(pdfengine_smoke PRIVATE skia::skia) endif() +if(PDFENGINE_WITH_PDFIUM) + target_link_libraries(pdfengine_smoke PRIVATE pdfium::pdfium) +endif() + if(PDFENGINE_WITH_QPDF) target_link_libraries(pdfengine_smoke PRIVATE qpdf::libqpdf ZLIB::ZLIB JPEG::JPEG) endif() diff --git a/gateway/Dockerfile b/gateway/Dockerfile index b577a8f..2b3e8f1 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,10 +1,9 @@ -FROM python:3.11-slim +# Stage 1: Builder +FROM python:3.11-slim AS builder ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ - PIP_NO_CACHE_DIR=1 \ - PIP_DISABLE_PIP_VERSION_CHECK=1 \ - PORT=8000 + VCPKG_FORCE_SYSTEM_BINARIES=1 RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -16,19 +15,99 @@ RUN apt-get update \ libjpeg-dev \ zlib1g-dev \ libpng-dev \ + curl \ + zip \ + unzip \ + tar \ + autoconf \ + autoconf-archive \ + automake \ + libtool \ + lld \ + && rm -rf /var/lib/apt/lists/* \ + && ln -sf /usr/bin/ld.lld /usr/bin/ld + +WORKDIR /build + +# Install vcpkg +RUN git clone https://github.com/microsoft/vcpkg.git /opt/vcpkg \ + && /opt/vcpkg/bootstrap-vcpkg.sh -disableMetrics +ENV VCPKG_ROOT=/opt/vcpkg + +# Cache vcpkg dependencies in a separate layer +COPY vcpkg.json ./ +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=cache,target=/opt/vcpkg/downloads \ + /opt/vcpkg/vcpkg install --triplet x64-linux + +# Install depot_tools globally with caching +RUN --mount=type=cache,target=/opt/depot_tools \ + if [ ! -d /opt/depot_tools/.git ]; then \ + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git /opt/depot_tools; \ + fi +ENV PATH="/opt/depot_tools:${PATH}" + +# Build PDFium for Linux (heavily cached, keeping the huge source tree out of the image layer) +COPY third_party/pdfium/ ./third_party/pdfium/ +RUN --mount=type=cache,target=/build/third_party/pdfium/checkout \ + ./third_party/pdfium/build_pdfium.sh + +# Copy everything needed for the engine and bindings build +COPY CMakeLists.txt CMakePresets.json ./ +COPY cmake/ ./cmake/ +COPY engine/ ./engine/ +COPY bindings/ ./bindings/ +COPY gateway/ ./gateway/ +COPY corpus/ ./corpus/ + +# Configure CMake with tests enabled +RUN --mount=type=cache,target=/root/.cache \ + --mount=type=cache,target=/opt/vcpkg/downloads \ + cmake --preset linux-release \ + -DPDFENGINE_BUILD_TESTS=ON \ + -DPDFENGINE_WITH_PDFIUM=ON \ + -DPDFENGINE_WITH_SKIA=OFF \ + -DPDFENGINE_WITH_QPDF=ON \ + -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake + +# Build all targets including tests and pdfengine_py +RUN cmake --build out/build/linux-release + +# Stage 2: Tester +FROM builder AS tester +RUN ctest --test-dir out/build/linux-release --output-on-failure + +# Stage 3: Runtime +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PORT=8000 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libjpeg62-turbo \ + zlib1g \ + libpng16-16 \ && rm -rf /var/lib/apt/lists/* +# Install the extension globally so it's not shadowed by the volume mount ./gateway:/home/app +COPY --from=tester /build/gateway/pdfengine*.so /usr/local/lib/python3.11/site-packages/ + RUN groupadd --system app \ && useradd --system --gid app --create-home --home-dir /home/app app WORKDIR /home/app -COPY pyproject.toml README.md ./ +# Copy gateway Python code +COPY gateway/pyproject.toml gateway/README.md ./ RUN python -m pip install --upgrade pip \ && pip install --no-cache-dir -e ".[dev]" -COPY app ./app -COPY tests ./tests +COPY gateway/app ./app +COPY gateway/tests ./tests RUN chown -R app:app /home/app USER app diff --git a/third_party/pdfium/build_pdfium.sh b/third_party/pdfium/build_pdfium.sh index aebdbba..57f25e1 100644 --- a/third_party/pdfium/build_pdfium.sh +++ b/third_party/pdfium/build_pdfium.sh @@ -35,8 +35,8 @@ CHECKOUT_DIR="${BUILD_ROOT}/checkout" echo ">> Build root: ${BUILD_ROOT}" # --- 1. Read and validate the pinned revision ------------------------------- -PDFIUM_REPO="$(grep -E '^PDFIUM_REPO=' "${PINNED_FILE}" | cut -d= -f2-)" -PDFIUM_COMMIT="$(grep -E '^PDFIUM_COMMIT=' "${PINNED_FILE}" | cut -d= -f2-)" +PDFIUM_REPO="$(grep -E '^PDFIUM_REPO=' "${PINNED_FILE}" | cut -d= -f2- | tr -d '\r')" +PDFIUM_COMMIT="$(grep -E '^PDFIUM_COMMIT=' "${PINNED_FILE}" | cut -d= -f2- | tr -d '\r')" if [[ -z "${PDFIUM_COMMIT}" || "${PDFIUM_COMMIT}" == "REPLACE_WITH_PINNED_COMMIT_SHA" ]]; then echo "ERROR: PDFium revision is not pinned. Edit pdfium.pinned first (see README.md)." >&2 exit 1 @@ -45,12 +45,14 @@ echo ">> PDFium pinned at ${PDFIUM_COMMIT}" # --- 2. depot_tools --------------------------------------------------------- mkdir -p "${BUILD_ROOT}" -if [[ ! -d "${DEPOT_TOOLS_DIR}" ]]; then - echo ">> Cloning depot_tools" - git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git \ - "${DEPOT_TOOLS_DIR}" +if ! command -v gclient &> /dev/null; then + if [[ ! -d "${DEPOT_TOOLS_DIR}/.git" ]]; then + echo ">> Cloning depot_tools" + git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git \ + "${DEPOT_TOOLS_DIR}" + fi + export PATH="${DEPOT_TOOLS_DIR}:${PATH}" fi -export PATH="${DEPOT_TOOLS_DIR}:${PATH}" # Do NOT set DEPOT_TOOLS_UPDATE=0 — depot_tools is designed to self-manage, and # on first use it must bootstrap. Reproducibility comes from the pinned PDFium # revision below, not from freezing depot_tools. @@ -58,9 +60,11 @@ export PATH="${DEPOT_TOOLS_DIR}:${PATH}" # Git settings injected per-process via GIT_CONFIG_* so the user's global git # config is never touched. core.autocrlf=false avoids gclient seeing dependency # checkouts as "uncommitted changes" on platforms where autocrlf is enabled. -export GIT_CONFIG_COUNT=2 +export GIT_CONFIG_COUNT=4 export GIT_CONFIG_KEY_0=core.autocrlf GIT_CONFIG_VALUE_0=false export GIT_CONFIG_KEY_1=core.filemode GIT_CONFIG_VALUE_1=false +export GIT_CONFIG_KEY_2=http.postBuffer GIT_CONFIG_VALUE_2=1048576000 +export GIT_CONFIG_KEY_3=core.compression GIT_CONFIG_VALUE_3=0 # --- 3. Fetch / sync the PDFium tree ---------------------------------------- mkdir -p "${CHECKOUT_DIR}"