From 14b94a668289d8dc42bc15beb55ec9b3877b870c Mon Sep 17 00:00:00 2001 From: Mohamed Gaber Date: Wed, 8 May 2024 15:41:43 +0300 Subject: [PATCH] Migrate Build System to Match OpenLane 2 (#2115) + Repository is now a Nix flake ~ Change all invocations of `openroad -python` to use `run_odbpy_script` for consistency ~ Change build system from ad-hoc to Nix, still producing a Docker image as a final result ~ Update KLayout scripts to use `klayout-pymod` or properly parse commandline arguments ~ `open_pdks` -> `bdc9412` to match OpenLane 2 - Remove local installer; `nix run .` will run OpenLane natively --- .github/actions/docker_build/action.yml | 41 +- .github/actions/set_env_variables/action.yml | 4 - .github/scripts/gh.py | 5 - .github/scripts/run_tests.py | 2 +- .github/workflows/openlane_ci.yml | 18 +- .github/workflows/tool_updater.yml | 82 --- Makefile | 6 +- README.md | 6 +- default.nix | 117 ++++ dependencies/arch/run_time.txt | 40 -- dependencies/centos-7/compile_time.txt | 69 --- dependencies/centos-7/precompile_time.txt | 2 - dependencies/centos-7/run_time.txt | 55 -- dependencies/hash_for.py | 53 -- dependencies/installer.py | 521 ------------------ dependencies/macos/run_time.txt | 40 -- dependencies/tool_metadata.yml | 68 +-- dependencies/ubuntu-20.04/run_time.txt | 48 -- dependencies/verify_versions.py | 80 +-- docker/.gitignore | 8 +- docker/Makefile | 118 +--- docker/README.md | 141 +++-- docker/build.sh | 12 + docker/build_base/.gitignore | 1 - docker/build_base/Dockerfile | 52 -- docker/current_platform.py | 9 +- docker/current_system.py | 9 + docker/docker.nix | 83 +++ docker/git/Dockerfile | 39 -- docker/klayout/Dockerfile | 43 -- docker/klayout/build_klayout.sh | 52 -- docker/magic/Dockerfile | 38 -- docker/netgen/Dockerfile | 38 -- docker/openlane/.gitignore | 1 - docker/openlane/Dockerfile.tpl | 62 --- docker/openroad_app/Dockerfile | 100 ---- docker/padring/Dockerfile | 40 -- docker/run_base/.dockerignore | 1 - docker/run_base/.gitignore | 5 - docker/run_base/Dockerfile | 36 -- docker/tar/.bashrc | 7 - docker/tar/.gitignore | 7 - docker/tar/.tclshrc | 3 - docker/utils.py | 381 ------------- docker/verilator/Dockerfile | 42 -- docker/vlogtoverilog/Dockerfile | 39 -- docker/yosys/Dockerfile | 41 -- docs/source/for_developers/gha_workflow.md | 8 +- docs/source/for_developers/tool_versions.md | 49 ++ .../getting_started/installation/index.rst | 1 - .../installation/installation_local.md | 32 -- docs/source/getting_started/quickstart.md | 4 - env.py | 8 - flake.lock | 123 +++++ flake.nix | 72 +++ flow.tcl | 12 +- gui.py | 14 +- scripts/klayout/open_design.py | 130 ++--- scripts/klayout/stream_out.py | 285 +++++----- scripts/odbpy/reader.py | 28 +- scripts/tcl_commands/all.tcl | 6 +- scripts/tcl_commands/floorplan.tcl | 6 +- scripts/tcl_commands/lvs.tcl | 2 +- scripts/utils/utils.tcl | 8 +- tests/1413-odb_remover/interactive.tcl | 20 +- 65 files changed, 844 insertions(+), 2629 deletions(-) delete mode 100644 .github/workflows/tool_updater.yml create mode 100644 default.nix delete mode 100644 dependencies/arch/run_time.txt delete mode 100644 dependencies/centos-7/compile_time.txt delete mode 100644 dependencies/centos-7/precompile_time.txt delete mode 100644 dependencies/centos-7/run_time.txt delete mode 100644 dependencies/hash_for.py delete mode 100644 dependencies/installer.py delete mode 100644 dependencies/macos/run_time.txt delete mode 100644 dependencies/ubuntu-20.04/run_time.txt create mode 100644 docker/build.sh delete mode 100644 docker/build_base/.gitignore delete mode 100644 docker/build_base/Dockerfile create mode 100644 docker/current_system.py create mode 100644 docker/docker.nix delete mode 100644 docker/git/Dockerfile delete mode 100644 docker/klayout/Dockerfile delete mode 100644 docker/klayout/build_klayout.sh delete mode 100644 docker/magic/Dockerfile delete mode 100644 docker/netgen/Dockerfile delete mode 100644 docker/openlane/.gitignore delete mode 100644 docker/openlane/Dockerfile.tpl delete mode 100644 docker/openroad_app/Dockerfile delete mode 100644 docker/padring/Dockerfile delete mode 100644 docker/run_base/.dockerignore delete mode 100644 docker/run_base/.gitignore delete mode 100644 docker/run_base/Dockerfile delete mode 100644 docker/tar/.bashrc delete mode 100644 docker/tar/.gitignore delete mode 100644 docker/tar/.tclshrc delete mode 100644 docker/utils.py delete mode 100644 docker/verilator/Dockerfile delete mode 100644 docker/vlogtoverilog/Dockerfile delete mode 100644 docker/yosys/Dockerfile create mode 100644 docs/source/for_developers/tool_versions.md delete mode 100644 docs/source/getting_started/installation/installation_local.md create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.github/actions/docker_build/action.yml b/.github/actions/docker_build/action.yml index 42e4620a0..737a95a4f 100644 --- a/.github/actions/docker_build/action.yml +++ b/.github/actions/docker_build/action.yml @@ -21,40 +21,33 @@ runs: using: "composite" steps: - uses: actions/checkout@v2 + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true - uses: docker/setup-buildx-action@v2 - uses: ./.github/actions/set_env_variables - - - name: Check If Going To Push An Image To Docker - shell: bash - run: | - export PUSHING=$(ruby -e 'if ("${{ github.event_name }}" != "pull_request" && "${{ inputs.dockerhub_user }}" != ""); print(1) else print(0) end') - echo "PUSHING=$PUSHING" >> $GITHUB_ENV - - - name: Login to DockerHub - if: ${{ env.PUSHING == '1' }} - uses: docker/login-action@v1 - with: - username: ${{ inputs.dockerhub_user }} - password: ${{ inputs.dockerhub_password }} - - name: Set up QEMU uses: docker/setup-qemu-action@v1 - + - uses: DeterminateSystems/nix-installer-action@main + with: + extra-conf: | + extra-substituters = https://openlane.cachix.org + extra-trusted-public-keys = openlane.cachix.org-1:qqdwh+QMNGmZAuyeQJTH9ErW57OWSvdtuwfBKdS254E= - name: Docker Build shell: bash run: | - export BUILD_IF_CANT_PULL=1 - export BUILD_COMMAND="docker build" - if [ "$PUSHING" = "1" ]; then - export BUILD_IF_CANT_PULL_THEN_PUSH=1 - export BUILD_COMMAND="docker buildx build --load --cache-from=type=gha --cache-to=type=gha,scope=${{ github.workflow }}}" - fi export BUILD_ARCH=${{ inputs.arch }} - cd docker/ && make merge - + make -C docker - name: Export Docker Image shell: bash - run: docker save -o /tmp/image-${{ inputs.arch }}.tar ${{ env.OPENLANE_IMAGE_NAME }}-${{ inputs.arch }} + run: | + docker save -o /tmp/image-${{ inputs.arch }}.tar efabless/openlane:current-${{ inputs.arch }} - name: Upload Docker Image uses: actions/upload-artifact@v2 diff --git a/.github/actions/set_env_variables/action.yml b/.github/actions/set_env_variables/action.yml index 71c457e86..555b61fb9 100644 --- a/.github/actions/set_env_variables/action.yml +++ b/.github/actions/set_env_variables/action.yml @@ -14,7 +14,3 @@ runs: - name: Export Branch Name shell: bash run: echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV - - - name: Export Temp Image Name - shell: bash - run: echo "OPENLANE_IMAGE_NAME=openlane:intermediate" >> $GITHUB_ENV diff --git a/.github/scripts/gh.py b/.github/scripts/gh.py index 861641a7d..b2b23fe7d 100644 --- a/.github/scripts/gh.py +++ b/.github/scripts/gh.py @@ -123,10 +123,6 @@ def export_env_alt(key, value): print('Environment variables required: "PDK_ROOT"') exit(os.EX_CONFIG) - if os.getenv("OPENLANE_IMAGE_NAME") is None: - print('Environment variables required: "OPENLANE_IMAGE_NAME"') - exit(os.EX_CONFIG) - origin = os.getenv("REPO_URL") repo = Repo("Openlane", origin) @@ -136,7 +132,6 @@ def export_env_alt(key, value): "run_id": os.getenv("GITHUB_RUN_ID"), "origin": origin, "branch": os.getenv("BRANCH_NAME"), - "image": os.getenv("OPENLANE_IMAGE_NAME"), "root": os.getenv("GITHUB_WORKSPACE"), "pdk_root": os.getenv("PDK_ROOT"), "pdk": os.getenv("PDK"), diff --git a/.github/scripts/run_tests.py b/.github/scripts/run_tests.py index c1c554afc..b4910a0c1 100644 --- a/.github/scripts/run_tests.py +++ b/.github/scripts/run_tests.py @@ -53,7 +53,7 @@ f"PDK_ROOT={gh.pdk_root}", "-e", f"PDK={gh.pdk}", - gh.image, + "efabless/openlane:current-amd64", "bash", "-c", shlex.join( diff --git a/.github/workflows/openlane_ci.yml b/.github/workflows/openlane_ci.yml index 6bc1046c5..b40a902ec 100644 --- a/.github/workflows/openlane_ci.yml +++ b/.github/workflows/openlane_ci.yml @@ -172,7 +172,9 @@ jobs: run: python3 -m pip install pyyaml - name: Run Issue Regression Test - run: cd ${GITHUB_WORKSPACE}/ && make run_issue_regression ISSUE_REGRESSION_DESIGN=${{ matrix.test }} + run: | + OPENLANE_IMAGE_NAME=efabless/openlane:current\ + make run_issue_regression ISSUE_REGRESSION_DESIGN=${{ matrix.test }} - name: Upload Logs if: ${{ always() }} @@ -231,7 +233,7 @@ jobs: - name: Run Test run: | - OPENLANE_IMAGE_NAME=$OPENLANE_IMAGE_NAME-amd64\ + OPENLANE_IMAGE_NAME=efabless/openlane:current-amd64\ python3 ${GITHUB_WORKSPACE}/.github/scripts/run_tests.py ${{ matrix.design.name }} - name: Escape Design Name @@ -267,6 +269,16 @@ jobs: with: fetch-depth: 0 + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + - name: Set up environment variables uses: ./.github/actions/set_env_variables @@ -355,7 +367,7 @@ jobs: run: | for tag in $TAG_LIST; do for arch in amd64 arm64v8; do - docker image tag ${{ env.OPENLANE_IMAGE_NAME }}-$arch ${{ vars.DOCKER_IMAGE }}:$tag-$arch + docker image tag efabless/openlane:current-$arch ${{ vars.DOCKER_IMAGE }}:$tag-$arch docker push ${{ vars.DOCKER_IMAGE }}:$tag-$arch done docker manifest create ${{ vars.DOCKER_IMAGE }}:$tag\ diff --git a/.github/workflows/tool_updater.yml b/.github/workflows/tool_updater.yml deleted file mode 100644 index 9426a5090..000000000 --- a/.github/workflows/tool_updater.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: Tool Updater - -on: - # Runs every day at midnight UTC - schedule: - - cron: "0 0 * * *" - # Manual Dispatch - workflow_dispatch: - -jobs: - create-cid-tools-branches: - runs-on: ubuntu-20.04 - strategy: - matrix: - tools: [magic, netgen, yosys, openroad_app, verilator] - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ vars.MAIN_BRANCH }} - - - name: Set up environment variables - uses: ./.github/actions/set_env_variables - - - name: Export TOOL Name - run: echo "TOOL=${{ matrix.tools }}" >> $GITHUB_ENV - - - name: Update TOOL - run: cd ${GITHUB_WORKSPACE}/ && python3 ${GITHUB_WORKSPACE}/.github/scripts/update_tools.py ${{ env.TOOL }} - - - name: Create Pull Request - if: ${{ env.NO_UPDATE != '1' }} - uses: peter-evans/create-pull-request@v4 - with: - token: ${{ secrets.MY_TOKEN }} - title: "[BOT] Update ${{ env.TOOL }}" - author: ${{ vars.BOT_AUTHOR_LINE }} - committer: ${{ vars.BOT_AUTHOR_LINE }} - body: | - This is an automated PR. - - See the individual commits for details. - commit-message: | - [BOT] Update ${{ env.TOOL }} - - ${{ env.TOOL }} -> ${{ env.TOOL_COMMIT_HASH }} - branch: ${{ matrix.tools }}-update-branch - push-to-fork: ${{ vars.FORK_NAME }} - delete-branch: true - create-cid-pdk-branch: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ vars.MAIN_BRANCH }} - - - name: Set up environment variables - uses: ./.github/actions/set_env_variables - - - name: Update PDK - run: python3 ${GITHUB_WORKSPACE}/.github/scripts/update_tools.py open_pdks - - - name: Create Pull Request - if: ${{ env.NO_UPDATE != '1' }} - uses: peter-evans/create-pull-request@v4 - with: - token: ${{ secrets.MY_TOKEN }} - title: "[BOT] Update PDK" - author: ${{ vars.BOT_AUTHOR_LINE }} - committer: ${{ vars.BOT_AUTHOR_LINE }} - body: | - This is an automated PR. - - See the individual commits for details. - commit-message: | - [BOT] Update PDK - - sky130 -> ${{ env.SKY130_COMMIT_HASH }} - open_pdks -> ${{ env.OPEN_PDKS_COMMIT_HASH }} - base: ${{ env.BRANCH_NAME }} - branch: pdk-update-branch - push-to-fork: ${{ vars.FORK_NAME }} - delete-branch: true diff --git a/Makefile b/Makefile index d24d487e3..85f226f29 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,6 @@ FASTEST_TEST_SET_TAG ?= FASTEST_TEST_SET EXTENDED_TEST_SET_TAG ?= EXTENDED_TEST_SET PRINT_REM_DESIGNS_TIME ?= 0 -SKYWATER_COMMIT ?= $(shell $(PYTHON_BIN) ./dependencies/tool.py sky130 -f commit) OPEN_PDKS_COMMIT ?= $(shell $(PYTHON_BIN) ./dependencies/tool.py open_pdks -f commit) export PDK_ROOT ?= $(HOME)/.volare @@ -114,8 +113,9 @@ ENV_COMMAND = $(ENV_START) $(OPENLANE_IMAGE_NAME)-$(DOCKER_ARCH) all: get-openlane pdk .PHONY: openlane -openlane: venv/created - @PYTHON_BIN=$(PWD)/venv/bin/$(PYTHON_BIN) $(MAKE) -C docker openlane +openlane: + @$(MAKE) -C docker openlane + docker tag efabless/openlane:current-$(DOCKER_ARCH) $(OPENLANE_IMAGE_NAME)-$(DOCKER_ARCH) .PHONY: openlane-and-push-tools openlane-and-push-tools: venv/created diff --git a/README.md b/README.md index d9d625416..4590f4ca4 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,9 @@

OpenLane

- Open in Colab License: Apache 2.0 - GitHub Actions Status Badge Documentation Build Status Badge - Invite to the Open Source Silicon Slack - Python Code Style: black + Invite to the Open Source Silicon Slack + Built with Nix

OpenLane is an automated RTL to GDSII flow based on several components including OpenROAD, Yosys, Magic, Netgen, CVC, SPEF-Extractor, KLayout and a number of custom scripts for design exploration and optimization. The flow performs all ASIC implementation steps from RTL all the way down to GDSII. diff --git a/default.nix b/default.nix new file mode 100644 index 000000000..d1cb214ad --- /dev/null +++ b/default.nix @@ -0,0 +1,117 @@ +# Copyright 2024 Efabless Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +{ + lib, + libparse, + stdenv, + python3, + makeWrapper, + ncurses, + coreutils-full, + gnugrep, + gnused, + gnutar, + gzip, + git, + bash, + klayout-pymod, + yosys, + opensta, + openroad, + klayout, + netgen, + magic, + verilog, + verilator, + tclFull, +}: +let + pyenv = (python3.withPackages (ps: with ps; [ + libparse + click + pyyaml + XlsxWriter + klayout-pymod + ])); + pyenv-sitepackages = "${pyenv}/${pyenv.sitePackages}"; +in + stdenv.mkDerivation rec { + name = "openlane1"; + + src = [ + ./flow.tcl + ./scripts + ./configuration + ./dependencies + ]; + + unpackPhase = '' + echo $src + for file in $src; do + BASENAME=$(python3 -c "import os; print('$file'.split('-', maxsplit=1)[1], end='$EMPTY')") + cp -r $file $PWD/$BASENAME + done + ls -lah + ''; + + passthru = { + pyenv = pyenv; + }; + + includedTools = [ + yosys + opensta + openroad + klayout + netgen + magic + verilog + verilator + tclFull + ]; + + propagatedBuildInputs = includedTools ++ [ + pyenv + ncurses + coreutils-full + gnugrep + gnused + bash + gnutar + gzip + git + ]; + + nativeBuildInputs = [makeWrapper]; + + installPhase = '' + mkdir -p $out/bin + cp -r * $out/bin + wrapProgram $out/bin/flow.tcl\ + --set PATH ${lib.makeBinPath (propagatedBuildInputs)}\ + --set PYTHONPATH ${pyenv-sitepackages} + ''; + + doCheck = true; + + computed_PATH = lib.makeBinPath (propagatedBuildInputs); + + meta = with lib; { + description = "RTL-to-GDSII flow for application-specific integrated circuits (ASIC)s"; + homepage = "https://efabless.com/openlane"; + mainProgram = "flow.tcl"; + license = licenses.asl20; + platforms = platforms.linux ++ platforms.darwin; + }; + } diff --git a/dependencies/arch/run_time.txt b/dependencies/arch/run_time.txt deleted file mode 100644 index 0057c260b..000000000 --- a/dependencies/arch/run_time.txt +++ /dev/null @@ -1,40 +0,0 @@ -https://aur.archlinux.org/tcllib.git - -autoconf -bison -cairo -clang -curl -eigen -flex -gawk -gcc -gdb -gettext -git -graphviz -help2man -klayout -libffi -libsm -libx11 -make -ncurses -neovim -ninja -patch -pcre2 -python -python-pip -readline -spdlog -strace -swig -tcl -tcsh -texinfo -tk -wget -xdot -xorg-server-xvfb -zlib diff --git a/dependencies/centos-7/compile_time.txt b/dependencies/centos-7/compile_time.txt deleted file mode 100644 index cdcf0d361..000000000 --- a/dependencies/centos-7/compile_time.txt +++ /dev/null @@ -1,69 +0,0 @@ -devtoolset-8 -devtoolset-8-libatomic-devel - -Xvfb -autoconf -automake -bison -boost169-devel -boost169-static -bzip2 -cairo -cairo-devel -clang -csh -curl -flex -gawk -gcc -gdb -gettext -gettext-devel -git -glibc-static -graphviz -help2man -libcurl-devel -libSM -libX11-devel -libXext -libXft -libffi -libffi-devel -libgomp -libjpeg-turbo-devel -libstdc++ -libstdc++-static -libxml2-devel -libxslt-devel -make -mesa-libGLU-devel -ncurses-devel -ninja-build -patch -pcre-devel -python36-devel -python36-libs -qt5-qtbase-devel -qt5-qtmultimedia-devel -qt5-qtxmlpatterns-devel -qt5-qtsvg-devel -qt5-qttools-devel -readline-devel -ruby-devel -strace -spdlog-devel -swig3 -tcl -tcl-devel -tcl-tclreadline-devel -tcllib -tclx -texinfo -tk -tk-devel -vim-common -wget -which -zlib-devel -zlib-static diff --git a/dependencies/centos-7/precompile_time.txt b/dependencies/centos-7/precompile_time.txt deleted file mode 100644 index 98d7827a4..000000000 --- a/dependencies/centos-7/precompile_time.txt +++ /dev/null @@ -1,2 +0,0 @@ -https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -centos-release-scl \ No newline at end of file diff --git a/dependencies/centos-7/run_time.txt b/dependencies/centos-7/run_time.txt deleted file mode 100644 index 5625973eb..000000000 --- a/dependencies/centos-7/run_time.txt +++ /dev/null @@ -1,55 +0,0 @@ -alsa-lib -cairo -gdb -gettext -git -libffi -libgomp -libjpeg -libSM -libXcursor -libXext -libXft -libXinerama -libXrandr -libyaml -make -mesa-libGLU -patch -pciutils -pciutils-libs -perl -perl-Switch -python36 -python36-pip -python36-tkinter -qt -qt5-qtbase -qt5-qtimageformats -qt5-qtmultimedia -qt5-qtxmlpatterns -qt5-qtsvg -qt5-qttools -qt5-qttools-libs-designer -qt5-qttools-libs-designercomponents -qt5-qttools-libs-help -qt-settings -qt-x11 -ruby -rubygem-io-console -rubygem-json -rubygem-psych -rubygem-rdoc -rubygems -ruby-irb -ruby-libs -strace -tcl -tcllib -tcl-tclreadline-devel -tclx -tk -wget -which -Xvfb -zlib \ No newline at end of file diff --git a/dependencies/hash_for.py b/dependencies/hash_for.py deleted file mode 100644 index 2852483e4..000000000 --- a/dependencies/hash_for.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2022 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Gets a hash of the current dependency set of a certain operating system. -""" - -import os -import sys -import glob -import hashlib - -__dir__ = os.path.dirname(os.path.abspath(__file__)) - - -def main(argv): - if len(argv) != 2: - print(f"Usage: {argv[0]} ", file=sys.stderr) - exit(64) - - os = argv[1] - if os not in ["macos", "centos-7", "ubuntu-20.04", "arch"]: - print(f"Unknown operating system pick '{os}'.") - exit(64) - - files = glob.glob(f"{__dir__}/python/*") - files += glob.glob(f"{__dir__}/{os}/*") - - files.sort() - - content = "" - for file in files: - content += open(file).read() - - hash = hashlib.sha256(content.encode("utf-8")).hexdigest() - - print(hash, end="") - - -if __name__ == "__main__": - main(sys.argv) diff --git a/dependencies/installer.py b/dependencies/installer.py deleted file mode 100644 index 8cdddf487..000000000 --- a/dependencies/installer.py +++ /dev/null @@ -1,521 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2021 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import os -import sys -import uuid -import tempfile -import pathlib -import textwrap -import subprocess -from os.path import join, abspath, dirname, exists, realpath -from typing import Tuple, Union, List - -openlane_dir = dirname(dirname(abspath(__file__))) -is_root = os.geteuid() == 0 - - -class chdir(object): - def __init__(self, path): - self.path = path - self.previous = None - - def __enter__(self): - self.previous = os.getcwd() - os.chdir(self.path) - - def __exit__(self, exc_type, exc_value, traceback): - os.chdir(self.previous) - if exc_type is not None: - raise exc_value - - -def sh(*args: Tuple[str], root: Union[bool, str] = False, **kwargs): - """ - args: shell arguments to run - root: - if False, the command will be executed as-is - if True, if the user is not root, "sudo" will be added to the command - if "retry", the command will be executed as-is first, and if it fails, - it is retried as root. - """ - args = list(args) - if root and not is_root: - args = ["sudo"] + args - try: - subprocess.run( - args, - check=True, - stderr=subprocess.PIPE if root == "retry" else None, - **kwargs, - ) - except subprocess.CalledProcessError as e: - if root == "retry": - args = ["sudo"] + args - subprocess.run(args, check=True, **kwargs) - else: - raise e - - -def download(url: str, ext: str) -> str: - path = f"/tmp/{uuid.uuid4()}.{ext}" - print(f"{url} -> {path}") - target = open(path, "wb") - sh("curl", "-L", url, stdout=target) - target.close() - return path - - -# Installer Class -class Installer(object): - def __init__(self): - self.envs: List[Tuple[str, str]] = [] - - def input_options(self, env: str, msg: str, options: List[str]) -> str: - value = None - env_value = os.getenv(env) - if env_value is not None and env_value.lower() in options: - value = env_value - else: - options_pretty = [] + options - options_pretty[0] = f"{options[0].upper()}" - value = input(f"{msg} [{'/'.join(options_pretty)}] > ") - if value == "": - value = options[0] - while value.lower() not in options: - value = input(f"Invalid input {value.lower()}, please retry: ") - - value = value.lower() - self.envs.append((env, value)) - return value - - def input_default(self, env: str, msg: str, default: str) -> str: - value = None - env_value = os.getenv(env) - if env_value is not None: - value = env_value - else: - value = input(f"{msg} [{default}] > ") - if value == "": - value = default - - self.envs.append((env, value)) - return value - - def run(self): - from dependencies.tool import Tool - from dependencies.get_tag import NoGitException, get_tag - from dependencies.env_info import OSInfo - - try: - import venv - except ImportError: - print( - "Python venv does not appear to be installed, and is required for local installations.", - file=sys.stderr, - ) - - try: - ol_version = get_tag() - except NoGitException: - print( - "Installing OpenLane locally requires a Git repository.", - file=sys.stderr, - ) - exit(-1) - - tools = Tool.from_metadata_yaml(open("./dependencies/tool_metadata.yml").read()) - - print( - textwrap.dedent( - """\ - OpenLane Local Installer - - Copyright 2021-2022 Efabless Corporation. Available under the Apache License, - Version 2.0. - - Ctrl+C at any time to quit. - - Make sure you read the documentation in ./docs/source/getting_started/installation_local.md - """ - ) - ) - - print( - "[ALERT] The local installer is no longer actively supported.\nSee https://github.com/The-OpenROAD-Project/OpenLane/issues/1300 for more info." - ) - - install_dir = realpath("./install") - - sh("mkdir", "-p", install_dir, root="retry") - - home_perms = os.stat(os.getenv("HOME")) - sh( - "chown", - "-R", - "%i:%i" % (home_perms.st_uid, home_perms.st_gid), - install_dir, - root="retry", - ) - - os_list = ["other", "ubuntu-20.04", "centos-7", "arch", "macos"] - - # Try to determine user's OS - def set_default_os(x): - os_list.insert(0, os_list.pop(os_list.index(x))) - - os_info = OSInfo.get() - - if os_info.distro == "macOS": - set_default_os("macos") - - if os_info.distro == "centos" and os_info.distro_version == "7": - set_default_os("centos-7") - - if os_info.distro == "ubuntu" and os_info.distro_version == "20.04": - set_default_os("ubuntu-20.04") - - if os_info.distro in ["manjaro", "arch"]: - set_default_os("arch") - - os_pick = self.input_options( - "OS", "Which UNIX/Unix-like OS are you using?", os_list - ) - - gcc_bin = os.getenv("CC") or "gcc" - gxx_bin = os.getenv("CXX") or "g++" - try: - if os_pick not in [ - "centos-7", - "macos", - ]: # The reason we ignore centos 7 and macos is that we're going to just use devtoolset-8/brew gcc anyway. - all_output = "" - try: - gcc_ver_output = subprocess.run( - [gcc_bin, "--version"], stdout=subprocess.PIPE - ) - all_output += gcc_ver_output.stdout.decode("utf8") - gx_ver_output = subprocess.run( - [gxx_bin, "--version"], stdout=subprocess.PIPE - ) - all_output += gx_ver_output.stdout.decode("utf8") - except Exception: - pass - if "clang" in all_output: - print( - textwrap.dedent( - f"""\ - We've detected that you're using Clang as your default C or C++ compiler. - Unfortunately, Clang is not compatible with some of the tools being - installed. - - You may continue this installation at your own risk, but we recommend - installing GCC. - - You can specify a compiler to use explicitly by invoking this script as - follows, for example: - - CC=/usr/local/bin/gcc-8 CXX=/usr/local/bin/g++-8 python3 {__file__} - """ - ) - ) - input( - "Press return if you understand the risk and wish to continue anyways >" - ) - except FileNotFoundError as e: - print(e, "(set as either CC or CXX)") - exit(os.EX_CONFIG) - - install_packages = "no" - if os_pick != "other": - install_packages = self.input_options( - "INSTALL_PACKAGES", - "Do you want to install dependencies using your package manager?", - ["no", "yes"], - ) - if install_packages != "no": - - def cat_all(dir): - result = "" - for file in os.listdir(dir): - result += open(join(dir, file)).read() - result += "\n" - return result - - if os_pick == "macos": - brew_packages = ( - cat_all(join(openlane_dir, "dependencies", "macos")) - .strip() - .split("\n") - ) - - sh("brew", "install", *brew_packages) - if os_pick == "centos-7": - yum_packages = ( - cat_all(join(openlane_dir, "dependencies", "centos-7")) - .strip() - .split("\n") - ) - - sh("yum", "install", "-y", *yum_packages, root="retry") - if os_pick == "arch": - raw = ( - cat_all(join(openlane_dir, "dependencies", "arch")) - .strip() - .split("\n") - ) - - arch_packages = [] - aur_packages = [] - - for entry in raw: - if entry.strip() == "": - continue - - if entry.startswith("https://"): - aur_packages.append(entry) - else: - arch_packages.append(entry) - - sh( - "pacman", - "-S", - "--noconfirm", - "--needed", - *arch_packages, - root="retry", - ) - - temp_dir = tempfile.gettempdir() - oaur_path = os.path.join(temp_dir, "openlane_aur") - pathlib.Path(oaur_path).mkdir(parents=True, exist_ok=True) - with chdir(oaur_path): - for package in aur_packages: - sh("rm", "-rf", "current") - sh("git", "clone", package, "current") - with chdir("current"): - sh("makepkg", "-si", "--noconfirm") - if os_pick == "ubuntu-20.04": - raw = ( - cat_all(join(openlane_dir, "dependencies", "ubuntu-20.04")) - .strip() - .split("\n") - ) - - apt_packages = [] - apt_debs = [] - - for entry in raw: - if entry.strip() == "": - continue - - if entry.startswith("https://"): - apt_debs.append(entry) - else: - apt_packages.append(entry) - sh("apt-get", "update", root="retry") - sh("apt-get", "install", "-y", "curl", root="retry") - for deb in apt_debs: - path = download(deb, "deb") - sh("apt-get", "install", "-y", "-f", path, root="retry") - sh("apt-get", "install", "-y", *apt_packages, root="retry") - - print("To re-run with the same options: ") - print(f"{' '.join(['%s=%s' % env for env in self.envs])} python3 {__file__}") - - run_env = os.environ.copy() - run_env["PREFIX"] = install_dir - run_env["PATH"] = f"{install_dir}/bin:{os.getenv('PATH')}" - - path_elements = ["$OL_INSTALL_DIR/venv/bin", "$OL_INSTALL_DIR/bin"] - - if os_pick == "centos-7": - run_env["CC"] = "/opt/rh/devtoolset-8/root/usr/bin/gcc" - run_env["CXX"] = "/opt/rh/devtoolset-8/root/usr/bin/g++" - run_env["PATH"] = f"/opt/rh/devtoolset-8/root/usr/bin:{os.getenv('PATH')}" - run_env[ - "LD_LIBRARY_PATH" - ] = f"/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:{os.getenv('LD_LIBRARY_PATH')}" - run_env[ - "CMAKE_INCLUDE_PATH" - ] = f"/usr/include/boost169:{os.getenv('CMAKE_INCLUDE_PATH')}" - run_env[ - "CMAKE_LIBRARY_PATH" - ] = f"/lib64/boost169:{os.getenv('CMAKE_LIBRARY_PATH')}" - elif os_pick == "macos": - - def get_prefix(tool): - return ( - subprocess.check_output(["brew", "--prefix", tool]) - .decode("utf8") - .strip() - ) - - klayout_app_path = self.input_default( - "KLAYOUT_MAC_APP", - "Please input the path to klayout.app (0.27.3 or later): ", - "/Applications/klayout.app", - ) - klayout_path_element = join(klayout_app_path, "Contents", "MacOS") - - run_env["CC"] = f"{get_prefix('gcc')}/bin/gcc-11" - run_env["CXX"] = f"{get_prefix('gcc')}/bin/g++-11" - run_env[ - "PATH" - ] = f"{get_prefix('swig@3')}/bin:{get_prefix('bison')}/bin:{get_prefix('flex')}/bin:{get_prefix('gnu-which')}/bin:{os.getenv('PATH')}" - run_env[ - "MAGIC_CONFIG_OPTS" - ] = f"--with-tcl={get_prefix('tcl-tk')} --with-tk={get_prefix('tcl-tk')}" - run_env["READLINE_CXXFLAGS"] = f"CXXFLAGS=-L{get_prefix('readline')}/lib" - - path_elements.append(f"{klayout_path_element}") - path_elements.append(f"{get_prefix('gnu-sed')}/libexec/gnubin") - path_elements.append(f"{get_prefix('bash')}/bin") - else: - run_env["CC"] = gcc_bin - self.envs.append(("CC", gcc_bin)) - run_env["CXX"] = gxx_bin - self.envs.append(("CXX", gxx_bin)) - - def copy(f): - sh("rm", "-rf", f) - sh("cp", "-r", join(openlane_dir, f), f) - - def install(): - print("Copying files...") - for folder in ["bin", "lib", "share", "build", "dependencies"]: - sh("mkdir", "-p", folder) - - print("Building Python virtual environment...") - venv_builder = venv.EnvBuilder(clear=True, with_pip=True) - venv_builder.create("./venv") - - pip_install_cmd = "python3 -m pip install --upgrade" - - subprocess.run( - [ - "bash", - "-c", - f""" - source ./venv/bin/activate - {pip_install_cmd} -r ../dependencies/python/precompile_time.txt - {pip_install_cmd} -r ../dependencies/python/compile_time.txt - {pip_install_cmd} -r ../dependencies/python/run_time.txt - pip3 install --upgrade volare - mkdir -p ./pdks - volare enable --pdk-root ./pdks {tools['open_pdks'].commit} - """, - ] - ) - - print("Building dependencies...") - with chdir("build"): - for folder in ["repos", "versions"]: - sh("mkdir", "-p", folder) - - skip_tools = re.compile(os.getenv("SKIP_TOOLS") or "Unmatchable") - tool_queue = list(tools.values()).copy() - - print(tool_queue) - - def pop(): - return tool_queue.pop(0) if len(tool_queue) else None - - installed = set() - tool = pop() - while tool is not None: - if not (tool.in_install and (skip_tools.match(tool.name) is None)): - tool = pop() - continue - - # if len(tool.dependencies): - # dependencies = set(tool.dependencies) - # if not dependencies.issubset(installed): - # tool_queue.append(tool) - # tool = pop() - # continue - - installed_version = "" - version_path = f"versions/{tool.name}" - try: - installed_version = open(version_path).read() - except Exception: - pass - if ( - installed_version == tool.version_string - and os.getenv("FORCE_REINSTALL") != "1" - ): - print(f"{tool.version_string} already installed, skipping...") - else: - print(f"Installing {tool.name}...") - - with chdir("repos"): - if not exists(tool.name): - sh("git", "clone", tool.repo, tool.name) - - with chdir(tool.name): - sh("git", "fetch") - sh("git", "checkout", tool.commit) - sh("git", "submodule", "update", "--init") - subprocess.run( - [ - "bash", - "-c", - f"""\ - set -e - source {install_dir}/venv/bin/activate - {tool.build_script} - """, - ], - env=run_env, - check=True, - ) - - with open(version_path, "w") as f: - f.write(tool.version_string) - - installed.add(tool.name) - tool = pop() - - path_elements.reverse() - with open("env.tcl", "w") as f: - f.write( - textwrap.dedent( - f"""\ - set OL_INSTALL_DIR [file dirname [file normalize [info script]]] - - set ::env(OPENLANE_LOCAL_INSTALL) 1 - set ::env(OL_INSTALL_DIR) "$OL_INSTALL_DIR" - set ::env(PATH) "{":".join(path_elements)}:$::env(PATH)" - set ::env(VIRTUAL_ENV) "$OL_INSTALL_DIR/venv" - if {{ ![info exists ::env(PDK_ROOT) ]}} {{ - set ::env(PDK_ROOT) "$OL_INSTALL_DIR/pdks" - }} - """ - ) - ) - - with open("installed_version", "w") as f: - f.write(ol_version) - - with chdir(install_dir): - install() - - print("Done.") - print( - "To invoke Openlane from now on, invoke ./flow.tcl from the OpenLane root without the Makefile." - ) diff --git a/dependencies/macos/run_time.txt b/dependencies/macos/run_time.txt deleted file mode 100644 index f8377401e..000000000 --- a/dependencies/macos/run_time.txt +++ /dev/null @@ -1,40 +0,0 @@ -autoconf -automake -bash -bison -boost -bzip2 -cairo -curl -eigen -flex -gawk -gcc@11 -gdb -gettext -graphviz -libSM -libX11 -libXext -libXft -libffi -libjpeg -libomp -libxml2 -libxslt -make -mesa-glu -ncurses -ninja -patchutils -pcre -python -readline -spdlog -swig@3 -tcl-tk -vim -wget -gnu-which -xdot -zlib diff --git a/dependencies/tool_metadata.yml b/dependencies/tool_metadata.yml index 94331dfe8..9d9b3ab5f 100644 --- a/dependencies/tool_metadata.yml +++ b/dependencies/tool_metadata.yml @@ -1,69 +1,3 @@ -- name: magic - repo: https://github.com/rtimothyedwards/magic - commit: 0afe4d87d4aacfbbb2659129a1858a22d216a920 - build: | - ./configure --prefix=$PREFIX $MAGIC_CONFIG_OPTS - make clean - make database/database.h - make -j$(nproc) - make install -- name: netgen - repo: https://github.com/rtimothyedwards/netgen - commit: 87d8759a6980d297edcb9be6f8661867e4726f9a - build: | - ./configure --prefix=$PREFIX $MAGIC_CONFIG_OPTS - make clean - make -j$(nproc) - make install -- name: padring - repo: https://github.com/donn/padring - commit: b2a64abcc8561d758c0bcb3945117dcb13bd9dca - build: | - bash ./bootstrap.sh - cd build - ninja clean - ninja - cp padring $PREFIX/bin -- name: vlogtoverilog - repo: https://github.com/RTimothyEdwards/qflow - commit: a550469b63e910ede6e3022e2886bca96462c540 - build: | - # Note that vlogtoverilog is part of the qflow repository. - ./configure - cd src - make clean - make -j$(nproc) vlog2Verilog - cp vlog2Verilog $PREFIX/bin -- name: verilator - repo: https://github.com/verilator/verilator - commit: 67dfa37c560385827218350ea936eb1baf604240 - build: | - autoconf - ./configure --prefix=$PREFIX - make - make install -- name: yosys - repo: https://github.com/YosysHQ/yosys - commit: 4a1b5599258881f579a2d95274754bcd8fc171bd - build: | - make clean - make PREFIX=$PREFIX config-gcc - make PREFIX=$PREFIX -j$(nproc) - make PREFIX=$PREFIX install -- name: klayout - repo: https://github.com/KLayout/klayout - commit: 44a2aa9ca17c2b1c154f9c410ded063de9ed3e12 - in_install: false -- name: openroad_app - repo: https://github.com/The-OpenROAD-Project/OpenROAD - commit: da0053d7b0014ab9c87ea148875ff6c2a0f9b658 - in_install: false -- name: git - repo: https://github.com/git/git - commit: e9d7761bb94f20acc98824275e317fa82436c25d - in_install: false - name: open_pdks repo: https://github.com/RTimothyEdwards/open_pdks - commit: cd1748bb197f9b7af62a54507de6624e30363943 - in_install: false - pdk: true + commit: bdc9412b3e468c102d01b7cf6337be06ec6e9c9a diff --git a/dependencies/ubuntu-20.04/run_time.txt b/dependencies/ubuntu-20.04/run_time.txt deleted file mode 100644 index b565ea90b..000000000 --- a/dependencies/ubuntu-20.04/run_time.txt +++ /dev/null @@ -1,48 +0,0 @@ -xvfb -autoconf -autopoint -bison -bzip2 -libcairo-dev -clang -csh -curl -flex -gawk -gcc -gdb -gettext -git -graphviz -help2man -libeigen3-dev -libsm-dev -libx11-dev -libffi-dev -libgomp1 -libjpeg9-dev -libxml2-dev -libxslt-dev -libffi-dev -libspdlog-dev -make -ncurses-dev -ninja-build -patch -libpcre2-dev -python3 -python3-pip -python3-dev -python3-tk -libreadline-dev -strace -swig -tcl-dev -tk-dev -tcllib -tclx8.4-dev -texinfo -neovim -wget -xdot -zlib1g diff --git a/dependencies/verify_versions.py b/dependencies/verify_versions.py index cdc81281c..07dfc4add 100644 --- a/dependencies/verify_versions.py +++ b/dependencies/verify_versions.py @@ -148,90 +148,12 @@ def verify_versions( print(traceback.format_exc(), file=report_file) raise Exception("Failed to compare PDKs.") - if not no_tools: - installed = os.getenv("OPENLANE_LOCAL_INSTALL") == "1" - environment_manifest = None - if installed: - # 3a. Compare with installed versions - installed_versions_path = join( - os.environ["OL_INSTALL_DIR"], "build", "versions" - ) - environment_manifest = [] - for tool in os.listdir(installed_versions_path): - protocol, url, commit = ( - open(join(installed_versions_path, tool)).read().split(":") - ) - repo = f"{protocol}:{url}" - environment_manifest.append( - {"name": tool, "repo": repo, "commit": commit} - ) - else: - # 3b. Compare Container And Installation Manifests - try: - container_manifest_path = join("/", "tool_metadata.yml") - environment_manifest = yaml.safe_load(open(container_manifest_path)) - except FileNotFoundError: - raise Exception( - "Container manifest not found. What this likely means is that the container is severely out of date." - ) - - tool_set_flow = ( - set([element["name"] for element in manifest]) - pdk_manifest_names - ) - tool_set_container = ( - set([element["name"] for element in environment_manifest]) - - pdk_manifest_names - ) - - unmatched_tools_flow = tool_set_flow - tool_set_container - for tool in unmatched_tools_flow: - tool_object = manifest_dict[tool] - if ( - tool_object.get("in_container") is not None - and not tool_object["in_container"] - ): - continue - if ( - installed - and tool_object.get("in_install") is not None - and not tool_object["in_install"] - ): - continue - print( - f"Tool {tool} is required by the flow scripts being used, but appears to not be installed in the environment.", - file=report_file, - ) - mismatches = True - - unmatched_tools_container = tool_set_container - tool_set_flow - for tool in unmatched_tools_container: - print( - f"Tool {tool} is installed in the environment, but has no corresponding entry in the flow scripts.", - file=report_file, - ) - mismatches = True - - for tool in environment_manifest: - if tool["name"] in pdk_manifest_names: - continue # PDK Stuff Already Checked - flow_script_counterpart = manifest_dict.get(tool["name"]) - if flow_script_counterpart is None: - continue - container_commit = tool["commit"] - flow_script_commit = flow_script_counterpart["commit"] - if container_commit != flow_script_commit: - print( - f"The version of {tool['name']} installed in the environment does not match the one required by the OpenLane flow scripts (installed: {container_commit}, expected: {flow_script_commit})", - file=report_file, - ) - mismatches = True - return mismatches if __name__ == "__main__": try: - no_tools = False + no_tools = True no_pdks = False mismatches = sys.argv[1] if mismatches == "none": diff --git a/docker/.gitignore b/docker/.gitignore index a4b093fe8..4b8c6285e 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -2,4 +2,10 @@ build/ logs/ */fetch_submodules_from_tarballs.py */utils.py -/base/ \ No newline at end of file +/base/ +tar/ +/openlane/ +ol2_hash +/openlane2 +/result +/git_version diff --git a/docker/Makefile b/docker/Makefile index 78543da40..4a7883389 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,117 +1,13 @@ -TOOL_REPOSITORY ?= efabless/openlane-tools - -include ../dependencies/image_name.mk - -BASE_HASH = $(shell $(PYTHON_BIN) ../dependencies/hash_for.py $(OS_NAME)) - PYTHON_BIN ?= python3 - -OS_NAME := centos-7 -OS_IMAGE := centos:centos7 -BASE_HASH := $(shell $(PYTHON_BIN) ../dependencies/hash_for.py $(OS_NAME)) BUILD_ARCH ?= $(shell $(PYTHON_BIN) ./current_platform.py) +NIX_SYSTEM := $(shell $(PYTHON_BIN) ./current_system.py $(BUILD_ARCH)) -export BUILD_BASE_TAG := build-base-$(BASE_HASH)-$(OS_NAME)-$(BUILD_ARCH) -export RUN_BASE_TAG := run-base-$(BASE_HASH)-$(OS_NAME)-$(BUILD_ARCH) -BUILD_BASE_IMAGE := $(TOOL_REPOSITORY):$(BUILD_BASE_TAG) -RUN_BASE_IMAGE := $(TOOL_REPOSITORY):$(RUN_BASE_TAG) - -# To use buildx: docker buildx build --push -export BUILD_COMMAND ?= docker build --no-cache - -TOOLS = $(shell $(PYTHON_BIN) ../dependencies/tool.py --containerized --no-pdks .) -OPENLANE_SKELETON=configuration dependencies designs/spm regression_results scripts AUTHORS.md env.py flow.tcl LICENSE run_designs.py -TOOL_BUILD_TARGETS = $(foreach tool,$(TOOLS),build-$(tool)) -TOOL_EXPORT_TARGETS = $(foreach tool,$(TOOLS),pull-$(tool)) - +HASH = $(shell git rev-parse HEAD) +HASH_SHORT = $(shell git rev-parse --short=7 HEAD) all: openlane +merge: openlane -# Build/Run Base Images -build-all: $(TOOL_BUILD_TARGETS) -build-build-base: ./build_base/Dockerfile - cat ../dependencies/centos-7/precompile_time.txt > ./build_base/yum_precompile_dependencies.txt - cat ../dependencies/centos-7/compile_time.txt > ./build_base/yum_compile_dependencies.txt - cat ../dependencies/centos-7/run_time.txt > ./build_base/yum_dependencies.txt - cat ../dependencies/python/precompile_time.txt > ./build_base/pip_precompile_dependencies.txt - cat ../dependencies/python/compile_time.txt > ./build_base/pip_compile_dependencies.txt - cat ../dependencies/python/run_time.txt > ./build_base/pip_dependencies.txt - mkdir -p logs - $(BUILD_COMMAND)\ - --build-arg OS_IMAGE=$(BUILD_ARCH)/$(OS_IMAGE)\ - -t $(BUILD_BASE_IMAGE)\ - build_base | tee logs/base.build.txt - -pull-build-base: - $(PYTHON_BIN) ./utils.py pull-if-doesnt-exist --repository $(TOOL_REPOSITORY) --os $(OS_NAME) --architecture $(BUILD_ARCH) build-base - @echo "-----------" - -# Used by pull-if-doesnt-exist -build-run-base: ./run_base/Dockerfile - cat ../dependencies/python/run_time.txt > ./run_base/pip_dependencies.txt - cat ../dependencies/centos-7/precompile_time.txt > ./run_base/yum_repos.txt - cat ../dependencies/centos-7/run_time.txt > ./run_base/yum_dependencies.txt - mkdir -p logs - $(BUILD_COMMAND)\ - --build-arg OS_IMAGE=$(BUILD_ARCH)/$(OS_IMAGE)\ - -t $(RUN_BASE_IMAGE)\ - run_base | tee logs/base.run.txt - -pull-run-base: - $(PYTHON_BIN) ./utils.py pull-if-doesnt-exist --repository $(TOOL_REPOSITORY) --os $(OS_NAME) --architecture $(BUILD_ARCH) run-base - @echo "-----------" - -# Tool Images -$(TOOL_BUILD_TARGETS): build-% : ./%/Dockerfile pull-build-base pull-run-base - mkdir -p logs - cp ./utils.py $* - $(BUILD_COMMAND)\ - `$(PYTHON_BIN) ../dependencies/tool.py --docker-args $*`\ - --build-arg RUN_BASE_IMAGE=$(RUN_BASE_IMAGE)\ - --build-arg BUILD_BASE_IMAGE=$(BUILD_BASE_IMAGE)\ - --target runnable\ - -t $(TOOL_REPOSITORY):`$(PYTHON_BIN) ../dependencies/tool.py --docker-tag-for-os=$(OS_NAME) --docker-arch=$(BUILD_ARCH) $*`\ - $* |\ - tee logs/$*.build.txt - -$(TOOL_EXPORT_TARGETS): pull-% : FORCE - $(PYTHON_BIN) ./utils.py pull-if-doesnt-exist --repository $(TOOL_REPOSITORY) --os $(OS_NAME) --architecture $(BUILD_ARCH) $* - @echo "-----------" - -# OpenLane Image -./tar/openlane: FORCE - rm -rf ./tar/openlane - mkdir -p ./tar/openlane - for file in $(OPENLANE_SKELETON); do\ - $(PYTHON_BIN) ./utils.py copy-tree --ignoring '*/runs' ../$$file ./tar/openlane/$$file ;\ - done - -.PHONY: merge openlane -openlane: merge -merge: pull-run-base $(TOOL_EXPORT_TARGETS) ./tar/openlane ../dependencies/tool_metadata.yml - cat ../dependencies/tool_metadata.yml > ./tar/tool_metadata.yml - printf "$(shell git rev-parse HEAD)" > ./tar/git_version - printf "$(shell git rev-parse --short=7 HEAD)" > ./tar/git_version_short - $(PYTHON_BIN) ./utils.py process-dockerfile-tpl --repository $(TOOL_REPOSITORY) --os $(OS_NAME) $(TOOLS) > ./openlane/Dockerfile - mkdir -p logs/tar - $(BUILD_COMMAND)\ - --build-arg ARCH=$(BUILD_ARCH)\ - --build-arg RUN_BASE_IMAGE=$(RUN_BASE_IMAGE)\ - -t $(OPENLANE_IMAGE_NAME)-$(BUILD_ARCH)\ - -f ./openlane/Dockerfile ./tar\ - | tee logs/$<.build.txt - rm -rf ./tar/openlane - -# Cleanup -.PHONY: clean -clean: clean_export clean_merge - -.PHONY: clean_merge -clean_merge: - rm -rf ./tar/openlane - -.PHONY: clean_export -clean_export: - rm -rf export/*.tar.gz - -FORCE: \ No newline at end of file +.PHONY: openlane +openlane: + NIX_SYSTEM=$(NIX_SYSTEM) BUILD_ARCH=$(BUILD_ARCH) bash build.sh diff --git a/docker/README.md b/docker/README.md index aeb4d91ba..a1b5fa109 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,66 +1,119 @@ -# Building and Customizing the Docker Image -## Structure +# OpenLane Docker Image -There are two "families" of images: one is for building tools, and the other is for running tools. +Since April 2024, the OpenLane build infrastructure has migrated to +[Nix](https://nixos.org), a declarative and reproducible build system. This was +done to streamline maintenance load. -The build family has a base image that contains all build dependencies and then a number of subimages named **builders** tasked with downloading all tools from source and building them. +This change has no bearing on the end-user experience, as it still +ultimately produces a Docker image. There are some minor adjustments, however: -The run family has a base image that contains all the running dependencies. There are a number of subimages named **runnables**, which copy the results from the builders and allows them to be runnable- those images are pushed to Docker Hub in the repository efabless/openlane-tools. Another image in the run family is the final OpenLane image, which has *all* the tools. +* Individual tools are no longer cached as a Docker image, instead residing in + [Cachix](https://cachix.org). +* The build scripts for the tools themselves reside in the OpenLane 2 repository + at . +* Support for `ppc64le`, which was never actively tested, has been dropped. -``` -openlane-build-base -L openroad_app **builder** -L [...] - -openlane-run-base -L openlane -L openroad_app **runnable** -``` +## For developers - Getting Nix -## Building the OpenLane Image -```bash -make # or make openlane # or make merge -``` +Follow the instructions at . -## Updating a Tool Binary +After everything is set up, from the repository's root directory, +`make openlane` as usual. -You can build a tool runnable using the following command: `make build-`. +## Overriding tool versions -When you are doing that, presumably you want to build a different version of the tool than is shipped by default with the OpenLane docker image. For example, suppose we would like to include OpenRoad's version of Yosys rather than one from https://github.com/yosyshq/yosys. We need to modify the file `tool_metadata.yml` that is located in `OpenLane/dependencies`. Here is an example of such a change: +As `tool_metadata.yml` no longer lists tools (only the PDK so Volare continues +to work,) overriding tool versions locally is done differently. -``` -- name: yosys - repo: https://github.com/The-OpenROAD-Project/yosys - commit: bc027b2cae9a85b887684930705762fac720b529 - build: | - make clean - make PREFIX=$PREFIX config-gcc - make PREFIX=$PREFIX -j$(nproc) - make PREFIX=$PREFIX install -``` +Tools included with OpenLane are listed in +https://github.com/efabless/openlane2/tree/overridable/nix. We've standardized +the function header to easily allow you to override the versions of tools. -Be alert to the fact that if you mix and match different tool versions (i.e. different git commit hashes), you can possibly run into compatibility issues. +Let's say, for instance, you'd like to override the revision for `magic`. First, +you'll check the `magic.nix` file for OpenLane 2: You'll find that the header +lists a `sha256` field in addition to either a `rev` (a git revision) or a +`version` (the version of the tool, minus any prefix `v`): -To list the available tools, `python3 ../dependencies/tool.py --containerized`, which is just essentially listing the contents of `tool_metadata.yml`. +```nix +{ + … + rev ? "bfd938b5e2321cf9a6c15f398fbc987b56fcc179", + sha256 ? "sha256-xNhPnNGoJ8YiG6NFeFhOuKTB56rQvggJugIvukao6U8=", +}: +``` -Be sure to `make openlane` **in the `docker/` folder** after building any tool, which will create the final OpenLane image. This will create a new Docker image tagged, by default, `efabless/openlane:current`. You can override the name as follows (again, **in the `docker/` folder**): +First, you'll open `flake.nix` and find a comment that says +`# ADD OVERRIDES HERE`. Below it, you'll add this override, with your desired +`rev` or `version` and sha256 equal to an empty string: -```sh - make openlane OPENLANE_IMAGE_NAME=whatever/whatever:whatever +```nix +magic = openlane2.packages."${pkgs.system}".magic.override { + rev = "ca99d0b76a82bc19a8b3213020ce3c135a28456e"; + sha256 = ""; +}; ``` -## Running the newly created Docker image -OpenLane scripts depend upon a variety of different shell environment variables in order to run correctly. They are all conveniently set by using the `make mount` command at the root of the repository. However, if you just do that, it will spin up the original image and not the one you just created. In order to use your new Docker image, first set the shell variable `OPENLANE_IMAGE_NAME` to the newly created image as follows (**in the root of the repository**): +Afterwards, invoke `nix build .#magic`. This will shortly fail with a message +that looks like this: -```sh - make mount OPENLANE_IMAGE_NAME=whatever/whatever:whatever +```log +error: hash mismatch in fixed-output derivation '/nix/store/i5pp79zmr4dngkr41z9ax75javz5456v-source.drv': + specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= + got: sha256-aFFKbSqIgpkYjFZfpW3C52N1yQc5+KiLyf5jC16K5UU= ``` -## Running as root -* For security reasons, we don't recommend the default root Docker installation on GNU/Linux. See https://docs.docker.com/engine/security/rootless/ for a safer Docker installation also supported by OpenLane. +You'll want to go ahead and update the override with the `got` value as follows: -By default `make mount` logs into the image with the user ID that is currently active. If you are running as an unprivileged user, you can use `make mount` to log in as root to the Docker image, but you will need to use `sudo` to do this. But, if you are depending on shell environment variables that you may have set during the current session they will be dropped by the `sudo` command. One way to pass those on to the sudo shell is to use the `-E` option. The following shows how you can do that: +```nix +magic = openlane2.packages."${pkgs.system}".magic.override { + rev = "ca99d0b76a82bc19a8b3213020ce3c135a28456e"; + sha256 = "sha256-aFFKbSqIgpkYjFZfpW3C52N1yQc5+KiLyf5jC16K5UU="; +}; ``` - sudo -E make mount + +And then invoke `nix build .#magic` again. Assuming the tool's build +instructions and dependencies have not changed, the build will be successful. +If the build instructions *have* changed, you will likely need to write a custom +Nix derivation for the tool in place of a simple override. + +But anyway, that's it. Afterwards, `make openlane` will create a Docker image +with your new utility. + +### Overriding versions for tools with dependencies + +Magic is a straightforward example, but some tools have dependencies on other +tools, chiefly, `openroad`, which depends on `openroad-abc` and `opensta`. + +In short, you will need to add overrides for all three tools as follows: + +```nix +opensta = openlane2.packages."${pkgs.system}".opensta.override { + rev = "a7f34210b403fe399c170296d54258f10f92885f"; + sha256 = "sha256-2R+ox0kcjXX5Kc6dtH/OEOccU/m8FjW1qnb0kxM/ahE="; +}; +openroad-abc = openlane2.packages."${pkgs.system}".openroad-abc.override { + rev = "d3916ac0337d599b30aeaf94e82b13338530ced3"; + sha256 = "sha256-osJzeOb0bgvbPGJjcpcfQzwcRJTZh1DYJ7RpFgw1NKg="; +}; +openroad = openlane2.packages."${pkgs.system}".openroad.override { + rev = "d423155d69de7f683a23f6916ead418a615ad4ad"; + sha256 = "sha256-RrJYdvzxD64TeNAlPs6G4BKxflpQO6ED78SqQVH7EUE="; + opensta = self.packages."${pkgs.system}".opensta; + openroad-abc = self.packages."${pkgs.system}".openroad-abc; +}; ``` + +Do note: + * You will need to do the whole `sha256` song and dance with all three. + * You need to list the dependencies explicitly in the overrides of the + dependent using the syntax shown for `openroad`. + +--- + +We understand this is more complex than the previous system; however +the benefits of using a build system and language designed specifically for +use-cases similar to ours far outweighs the added complexity: including better +caching and reproducibility. + +If you require any help overriding a tool, please feel free to open an issue. diff --git a/docker/build.sh b/docker/build.sh new file mode 100644 index 000000000..cae9e782e --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,12 @@ +set -x +set -e +TARBALL=$(nix build\ + --no-link\ + --print-out-paths\ + --accept-flake-config\ + --option system $NIX_SYSTEM\ + --extra-platforms $NIX_SYSTEM\ + ..#packages.$NIX_SYSTEM.openlane1-docker\ +) +cat $TARBALL | docker load +nix store delete $TARBALL diff --git a/docker/build_base/.gitignore b/docker/build_base/.gitignore deleted file mode 100644 index 314f02b1b..000000000 --- a/docker/build_base/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.txt \ No newline at end of file diff --git a/docker/build_base/Dockerfile b/docker/build_base/Dockerfile deleted file mode 100644 index 5a94c2c52..000000000 --- a/docker/build_base/Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2020-2021 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ARG OS_IMAGE= -FROM ${OS_IMAGE} - -# Install Yum Dependencies -COPY ./yum_precompile_dependencies.txt /yum_precompile_dependencies.txt -COPY ./yum_compile_dependencies.txt /yum_compile_dependencies.txt -COPY ./yum_dependencies.txt /yum_dependencies.txt -RUN yum install --setopt=skip_missing_names_on_install=False -y $(cat /yum_precompile_dependencies.txt | tr '\n' ' ') && \ - yum install --setopt=skip_missing_names_on_install=False -y $(cat /yum_compile_dependencies.txt | tr '\n' ' ') && \ - yum install --setopt=skip_missing_names_on_install=False -y $(cat /yum_dependencies.txt | tr '\n' ' ') && \ - yum clean all && \ - rm -rf /var/cache/yum - -RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 60 - -# Configure Git -RUN git config --global user.name "OpenLane User" -RUN git config --global user.email "openlane.user@localhost" - -ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc \ - CPP=/opt/rh/devtoolset-8/root/usr/bin/cpp \ - CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ \ - PATH=/opt/rh/devtoolset-8/root/usr/bin:$PATH \ - LD_LIBRARY_PATH=/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:$LD_LIBRARY_PATH - -ENV LD_LIBRARY_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64:/opt/rh/llvm-toolset-7.0/root/usr/lib:/opt/rh/llvm-toolset-7.0/root/usr/lib64/dyninst:/opt/rh/llvm-toolset-7.0/root/usr/lib/dyninst:/opt/rh/llvm-toolset-7.0/root/usr/lib64:/opt/rh/llvm-toolset-7.0/root/usr/lib:$LD_LIBRARY_PATH - -# Install Python Dependencies -## Wheel Build Dependencies -RUN python3 -m pip install --no-cache-dir --upgrade pip -COPY ./pip_compile_dependencies.txt /pip_compile_dependencies.txt -COPY ./pip_dependencies.txt /pip_dependencies.txt -RUN python3 -m pip install --no-cache-dir -r /pip_compile_dependencies.txt -RUN python3 -m pip install --no-cache-dir -r /pip_dependencies.txt - -# Update Certs -RUN yum update -y ca-certificates - -# Environment Configuration diff --git a/docker/current_platform.py b/docker/current_platform.py index ed262db3c..b15d2801f 100644 --- a/docker/current_platform.py +++ b/docker/current_platform.py @@ -1,4 +1,3 @@ -import sys import platform @@ -9,14 +8,8 @@ def current_docker_platform() -> str: return "amd64" elif arch in ["aarch64", "arm64"]: return "arm64v8" - elif arch in ["ppc64le"]: - return "ppc64le" else: - print( - f"Unsupported architecture '{platform.machine()}' Falling back to x86-64 for Docker.", - file=sys.stderr, - ) - return "amd64" + raise ValueError(f"Unsupported platform {arch}") if __name__ == "__main__": diff --git a/docker/current_system.py b/docker/current_system.py new file mode 100644 index 000000000..738433693 --- /dev/null +++ b/docker/current_system.py @@ -0,0 +1,9 @@ +import sys + + +def current_nix_system(arch: str) -> str: + return {"amd64": "x86_64-linux", "arm64v8": "aarch64-linux"}[arch] + + +if __name__ == "__main__": + print(current_nix_system(*sys.argv[1:]), end="") diff --git a/docker/docker.nix b/docker/docker.nix new file mode 100644 index 000000000..fd2793e27 --- /dev/null +++ b/docker/docker.nix @@ -0,0 +1,83 @@ +# Copyright 2023 Efabless Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +{ + dockerTools, + createDockerImage, + system, + pkgs, + lib, + python3, + openlane1, + git, + neovim, + zsh, + silver-searcher, + coreutils, +}: + +assert builtins.elem system ["x86_64-linux" "aarch64-linux"]; + +let + docker-arch-name = if system == "x86_64-linux" then + "amd64" + else + "arm64v8" + ; +in (createDockerImage { + inherit pkgs; + inherit lib; + name = "efabless/openlane"; + tag = "current-${docker-arch-name}"; + extraPkgs = with dockerTools; [ + git + zsh + neovim + silver-searcher + + openlane1 + openlane1.pyenv + ]; + nixConf = { + extra-experimental-features = "nix-command flakes repl-flake"; + }; + maxLayers = 2; + channelURL = "https://nixos.org/channels/nixos-23.11"; + + image-created = "now"; + image-extraCommands = '' + mkdir -p ./etc + cp -r ${openlane1}/bin ./openlane1 + chmod -R 755 ./openlane1 + cat < ./etc/zshrc + autoload -U compinit && compinit + autoload -U promptinit && promptinit && prompt suse && setopt prompt_sp + autoload -U colors && colors + + export PS1=$'%{\033[31m%}OpenLane Container%{\033[0m%}:%{\033[32m%}%~%{\033[0m%}%% '; + HEREDOC + ''; + image-config-cwd = "/openlane"; + image-config-cmd = ["${zsh}/bin/zsh"]; + image-config-extra-path = [ + "/openlane" + openlane1.computed_PATH + ]; + image-config-extra-env = [ + "LANG=C.UTF-8" + "LC_ALL=C.UTF-8" + "LC_CTYPE=C.UTF-8" + "EDITOR=nvim" + "TMPDIR=/tmp" + ]; +}) diff --git a/docker/git/Dockerfile b/docker/git/Dockerfile deleted file mode 100644 index 59cab5882..000000000 --- a/docker/git/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2020-2021 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ARG RUN_BASE_IMAGE= -ARG BUILD_BASE_IMAGE= -FROM ${BUILD_BASE_IMAGE} AS builder - -ARG GIT_REPO -ARG GIT_COMMIT - -WORKDIR /git -RUN curl -L ${GIT_REPO}/tarball/${GIT_COMMIT} |\ - tar -xzC . --strip-components=1 &&\ - make configure &&\ - ./configure --prefix=/build &&\ - make -j$(nproc)&&\ - make install - -RUN mkdir -p /build/version/ -RUN date +"Build Timestamp: %Y-%m-%d_%H-%M-%S" > /build/version/git.version -RUN echo ${GIT_COMMIT} >> /build/version/git.version -RUN tar -czf /build.tar.gz /build - -# --- -FROM ${RUN_BASE_IMAGE} AS runnable -ENV PATH /build/bin:$PATH - -COPY --from=builder /build /build -COPY --from=builder /build.tar.gz /build.tar.gz diff --git a/docker/klayout/Dockerfile b/docker/klayout/Dockerfile deleted file mode 100644 index e4ea97727..000000000 --- a/docker/klayout/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2020-2021 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG RUN_BASE_IMAGE= -ARG BUILD_BASE_IMAGE= -FROM ${BUILD_BASE_IMAGE} AS builder - -# git clone yosys -ARG KLAYOUT_REPO -ARG KLAYOUT_COMMIT - -RUN mkdir -p /build - -WORKDIR /klayout -RUN curl -L ${KLAYOUT_REPO}/tarball/${KLAYOUT_COMMIT} | tar -xzC . --strip-components=1 - -COPY ./build_klayout.sh ./build_klayout.sh - -RUN bash ./build_klayout.sh - -RUN mkdir -p /build/version/ - -RUN date +"Build Timestamp: %Y-%m-%d_%H-%M-%S" > /build/version/klayout.version -RUN echo ${KLAYOUT_COMMIT} >> /build/version/klayout.version -RUN tar -czf /build.tar.gz /build - -# --- -FROM ${RUN_BASE_IMAGE} AS runnable -ENV PATH /build/bin:$PATH - -COPY --from=builder /build /build -COPY --from=builder /build.tar.gz /build.tar.gz diff --git a/docker/klayout/build_klayout.sh b/docker/klayout/build_klayout.sh deleted file mode 100644 index 83ca2f934..000000000 --- a/docker/klayout/build_klayout.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -set -e -set -x - -bininstdir="./tmp/bin" -builddir="./tmp/build" - -# destination folders -bindir="/build/bin" -libdir="/build/lib/klayout" - -distpackdir="/build/lib/python3/dist-packages" -pylibdir="$distpackdir/klayout" - -# clean bin directory -rm -rf $bininstdir - -sed -i 's/-O2 -g/-O2/' /usr/lib64/qt5/mkspecs/linux-g++/qmake.conf - -# do the actual build -./build.sh\ - -qt5 \ - -j$(nproc) \ - -without-qtbinding \ - -bin $bininstdir \ - -build $builddir \ - -rpath $libdir - -echo "Copying files..." - -mkdir -p ${libdir}/db_plugins -mkdir -p ${libdir}/lay_plugins -mkdir -p ${pylibdir} -mkdir -p ${bindir} - -cp -pd $bininstdir/strm* ${bindir} -cp -pd $bininstdir/klayout ${bindir} -cp -pd $bininstdir/lib*so* ${libdir} -cp -pd $bininstdir/db_plugins/lib*so* ${libdir}/db_plugins -cp -pd $bininstdir/lay_plugins/lib*so* ${libdir}/lay_plugins -cp -pd $bininstdir/pymod/klayout/*so ${pylibdir} -cp -pd $bininstdir/pymod/klayout/*py ${pylibdir} -for d in db tl rdb lib; do - mkdir -p ${pylibdir}/$d - cp -pd $bininstdir/pymod/klayout/$d/*py ${pylibdir}/$d -done - -echo "Stripping..." -strip ${bindir}/* -strip ${pylibdir}/*.so -strip ${libdir}/db_plugins/*.so* -strip ${libdir}/lay_plugins/*.so* \ No newline at end of file diff --git a/docker/magic/Dockerfile b/docker/magic/Dockerfile deleted file mode 100644 index 293ab4051..000000000 --- a/docker/magic/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2020-2021 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ARG RUN_BASE_IMAGE= -ARG BUILD_BASE_IMAGE= -FROM ${BUILD_BASE_IMAGE} AS builder - -# Upstream is git://opencircuitdesign.com/magic, but servers are not stable enough for CI. -ARG MAGIC_REPO -ARG MAGIC_COMMIT - -WORKDIR /magic -RUN curl -L ${MAGIC_REPO}/tarball/${MAGIC_COMMIT} | tar -xzC . --strip-components=1 && \ - ./configure --prefix=/build && \ - make -j$(nproc) && \ - make install - -RUN mkdir -p /build/version/ -RUN date +"Build Timestamp: %Y-%m-%d_%H-%M-%S" > /build/version/magic.version -RUN echo ${MAGIC_COMMIT} >> /build/version/magic.version -RUN tar -czf /build.tar.gz /build - -# --- -FROM ${RUN_BASE_IMAGE} AS runnable -ENV PATH /build/bin:$PATH - -COPY --from=builder /build /build -COPY --from=builder /build.tar.gz /build.tar.gz \ No newline at end of file diff --git a/docker/netgen/Dockerfile b/docker/netgen/Dockerfile deleted file mode 100644 index 51014a245..000000000 --- a/docker/netgen/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2020-2021 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ARG RUN_BASE_IMAGE= -ARG BUILD_BASE_IMAGE= -FROM ${BUILD_BASE_IMAGE} AS builder - -# Upstream is git://opencircuitdesign.com/netgen, but servers are not stable enough for CI. -ARG NETGEN_REPO -ARG NETGEN_COMMIT - -WORKDIR /netgen - -RUN curl -L ${NETGEN_REPO}/tarball/${NETGEN_COMMIT} | tar -xzC . --strip-components=1 && \ - ./configure CFLAGS="-O2 -g" --prefix=/build && \ - make -j$(nproc) && \ - make install - -RUN mkdir -p /build/version/ -RUN date +"Build Timestamp: %Y-%m-%d_%H-%M-%S" > /build/version/netgen.version -RUN echo ${NETGEN_COMMIT} >> /build/version/netgen.version -RUN tar -czf /build.tar.gz /build -# --- -FROM ${RUN_BASE_IMAGE} AS runnable -ENV PATH /build/bin:$PATH - -COPY --from=builder /build /build -COPY --from=builder /build.tar.gz /build.tar.gz diff --git a/docker/openlane/.gitignore b/docker/openlane/.gitignore deleted file mode 100644 index 51cc96c8d..000000000 --- a/docker/openlane/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/Dockerfile \ No newline at end of file diff --git a/docker/openlane/Dockerfile.tpl b/docker/openlane/Dockerfile.tpl deleted file mode 100644 index 000c2cde0..000000000 --- a/docker/openlane/Dockerfile.tpl +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2020-2022 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG ARCH= -ARG RUN_BASE_IMAGE= -# -FROM ${RUN_BASE_IMAGE} - -# Environment Configuration -ENV OPENLANE_ROOT=/openlane -ENV OPENROAD_BIN openroad - -ENV OPENROAD=/build/ -ENV PATH=$OPENLANE_ROOT:$OPENLANE_ROOT/scripts:$OPENROAD/bin:$OPENROAD/bin/Linux-x86_64:$OPENROAD/pdn/scripts:$PATH -ENV PYTHONPATH=$PYTHONPATH:/build/lib/python3/dist-packages -ENV LD_LIBRARY_PATH=$OPENROAD/lib:$OPENROAD/lib/Linux-x86_64:$LD_LIBRARY_PATH -ENV MANPATH=$OPENROAD/share/man:$MANPATH -ENV PDK_ROOT /build/pdk - -# Locale -RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 -ENV LC_CTYPE en_US.UTF-8 - -# Tools -## Graphical Applications -RUN dbus-uuidgen --ensure - -## Copy manifest -ADD ./tool_metadata.yml /tool_metadata.yml - -## Copy Version -ADD ./git_version /git_version -ADD ./git_version_short /git_version_short - -## Scripts and Binaries -COPY ./openlane /openlane -# - -## Tclsh RC -COPY ./.tclshrc /.tclshrc -COPY ./.tclshrc /root/.tclshrc - -## Bash RC -COPY ./.bashrc /.bashrc -COPY ./.bashrc /root/.bashrc - -WORKDIR $OPENLANE_ROOT - -CMD /bin/bash diff --git a/docker/openroad_app/Dockerfile b/docker/openroad_app/Dockerfile deleted file mode 100644 index 10a5ec552..000000000 --- a/docker/openroad_app/Dockerfile +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2020-2021 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG RUN_BASE_IMAGE= -ARG BUILD_BASE_IMAGE= -FROM ${BUILD_BASE_IMAGE} AS builder - -# Build Boost -WORKDIR /boost -RUN curl -L https://sourceforge.net/projects/boost/files/boost/1.80.0/boost_1_80_0.tar.bz2/download | tar --strip-components=1 -xjC . && \ - ./bootstrap.sh && \ - ./b2 install --with-iostreams --with-test --with-serialization --with-system --with-thread -j $(nproc) - -# Build Eigen -WORKDIR /eigen -RUN curl -L https://gitlab.com/libeigen/eigen/-/archive/3.3/eigen-3.3.tar.gz | tar --strip-components=1 -xzC . && \ - mkdir build && \ - cd build && \ - cmake .. && \ - make -j$(nproc) && \ - make install - -# Build Lemon -WORKDIR /lemon -RUN curl -L https://github.com/The-OpenROAD-Project/lemon-graph/archive/refs/tags/1.3.1.tar.gz | tar --strip-components=1 -xzC . && \ - cmake -B build . && \ - cmake --build build -j $(nproc) --target install - -# Build Spdlog -WORKDIR /spdlog -RUN curl -L https://github.com/gabime/spdlog/archive/refs/tags/v1.8.1.tar.gz | tar --strip-components=1 -xzC . && \ - mkdir build && \ - cd build && \ - cmake .. && \ - make install -j $(nproc) - -# Build Swig -RUN yum remove -y swig3 -WORKDIR /swig -RUN curl -L https://github.com/swig/swig/archive/refs/tags/v4.0.1.tar.gz | tar --strip-components=1 -xzC . && \ - ./autogen.sh && \ - ./configure --prefix=/usr && \ - make -j $(nproc) && \ - make install - -# Build OR Tools -WORKDIR /ortools -RUN curl -L https://github.com/google/or-tools/archive/refs/tags/v9.4.tar.gz | tar --strip-components=1 -xzC . && \ - cmake -B build . -DBUILD_DEPS:BOOL=ON && \ - cmake --build build -j $(nproc) --target install - -# Set up OpenROAD Build Files -RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 -ENV LC_CTYPE en_US.UTF-8 - -ARG OPENROAD_APP_REPO -ARG OPENROAD_APP_COMMIT - -WORKDIR /openroad -RUN curl -L ${OPENROAD_APP_REPO}/tarball/${OPENROAD_APP_COMMIT} | tar -xzC . --strip-components=1 -COPY ./utils.py . -RUN python3 ./utils.py fetch-submodules-from-tarballs ${OPENROAD_APP_REPO} ${OPENROAD_APP_COMMIT} - -# Build OpenROAD -RUN sed -i "s/GITDIR-NOTFOUND/${OPENROAD_APP_COMMIT}/" cmake/GetGitRevisionDescription.cmake -RUN sed -i "s/-m64//" src/par/CMakeLists.txt -RUN sed -i "s/static char Cases/static signed char Cases/" third-party/abc/src/misc/extra/extraUtilMisc.c -RUN mkdir build && mkdir -p /build/version && mkdir install -RUN cd build && cmake -DCMAKE_INSTALL_PREFIX=$(pwd)/install .. -RUN cd build && make -j$(nproc) -RUN cd build && make install -RUN cp -r build/install/bin /build/ - -## Add OR_Tools -RUN mkdir -p /build/lib -RUN cp /usr/local/lib64/libortools.so.9 /build/lib/libortools.so.9 - -RUN date +"Build Timestamp: %Y-%m-%d_%H-%M-%S" > /build/version/openroad.version -RUN echo ${OPENROAD_APP_COMMIT} >> /build/version/openroad.version -RUN tar -czf /build.tar.gz /build - -# --- -FROM ${RUN_BASE_IMAGE} AS runnable -ENV PATH /build/bin:$PATH - -COPY --from=builder /build /build -COPY --from=builder /build.tar.gz /build.tar.gz diff --git a/docker/padring/Dockerfile b/docker/padring/Dockerfile deleted file mode 100644 index 6758dcd9a..000000000 --- a/docker/padring/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2020-2021 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ARG RUN_BASE_IMAGE= -ARG BUILD_BASE_IMAGE= -FROM ${BUILD_BASE_IMAGE} AS builder - -ARG PADRING_REPO -ARG PADRING_COMMIT -WORKDIR /padring - -RUN curl -L ${PADRING_REPO}/tarball/${PADRING_COMMIT} | tar -xzC . --strip-components=1 && \ - ./bootstrap.sh && \ - cd build/ && \ - ninja-build && \ - mkdir /build/bin/ -p && \ - cp padring /build/bin - -RUN mkdir -p /build/version -RUN date +"Build Timestamp: %Y-%m-%d_%H-%M-%S" > /build/version/padring.version -RUN echo ${PADRING_COMMIT} >> /build/version/padring.version -RUN tar -czf /build.tar.gz /build - -# --- -ARG RUN_BASE_IMAGE= -FROM ${RUN_BASE_IMAGE} AS runnable -ENV PATH /build/bin:$PATH - -COPY --from=builder /build /build -COPY --from=builder /build.tar.gz /build.tar.gz diff --git a/docker/run_base/.dockerignore b/docker/run_base/.dockerignore deleted file mode 100644 index 51cc96c8d..000000000 --- a/docker/run_base/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -/Dockerfile \ No newline at end of file diff --git a/docker/run_base/.gitignore b/docker/run_base/.gitignore deleted file mode 100644 index bbb69ec01..000000000 --- a/docker/run_base/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/*.tar.gz -/*.yml -/*.txt -/git_version -/git_version_short \ No newline at end of file diff --git a/docker/run_base/Dockerfile b/docker/run_base/Dockerfile deleted file mode 100644 index 4cfecad67..000000000 --- a/docker/run_base/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2020-2021 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ARG OS_IMAGE= -FROM ${OS_IMAGE} - -# Install Yum Dependencies -COPY ./yum_dependencies.txt /yum_dependencies.txt -COPY ./yum_repos.txt /yum_repos.txt -RUN yum install --setopt=skip_missing_names_on_install=False -y $(cat /yum_repos.txt) && \ - yum install --setopt=skip_missing_names_on_install=False -y $(cat /yum_dependencies.txt) && \ - yum clean all && \ - rm -rf /var/cache/yum - -RUN alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 60 - -# Configure Git -RUN git config --global user.name "OpenLane User" -RUN git config --global user.email "openlane.user@localhost" - -# Install Python Dependencies -## Wheel Build Dependencies -RUN python3 -m pip install --no-cache-dir --upgrade pip -COPY ./pip_dependencies.txt /pip_dependencies.txt -RUN yum install -y swig3 gcc gcc-c++ python3-devel -RUN python3 -m pip install --no-cache-dir -r /pip_dependencies.txt \ No newline at end of file diff --git a/docker/tar/.bashrc b/docker/tar/.bashrc deleted file mode 100644 index 5ae452038..000000000 --- a/docker/tar/.bashrc +++ /dev/null @@ -1,7 +0,0 @@ -# OpenLane .bashrc file -# Source global definitions -alias ll='ls -lAFh'; - -export OL_GIT_VERSION=$(cat /git_version_short); - -export PS1="\[\033[1;31m\]OpenLane Container ($OL_GIT_VERSION)\[\033[0m\]:\[\033[4;32m\]\w\[\033[0m\]$ "; \ No newline at end of file diff --git a/docker/tar/.gitignore b/docker/tar/.gitignore deleted file mode 100644 index 8040e6214..000000000 --- a/docker/tar/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -/*.tar.gz -/*.yml -/*.txt -/git_version -/git_version_short -/build -/openlane \ No newline at end of file diff --git a/docker/tar/.tclshrc b/docker/tar/.tclshrc deleted file mode 100644 index 3eb52551a..000000000 --- a/docker/tar/.tclshrc +++ /dev/null @@ -1,3 +0,0 @@ -if { [info exists ::env(OPENLANE_ROOT)] } { - package require openlane -} diff --git a/docker/utils.py b/docker/utils.py deleted file mode 100644 index a40614832..000000000 --- a/docker/utils.py +++ /dev/null @@ -1,381 +0,0 @@ -# Copyright 2021 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import re -import os -import sys -import json -import click -import shutil -import fnmatch -import pathlib -import tempfile -import subprocess -import urllib.error -import urllib.parse -import urllib.request - -SUPPORTED_ARCHITECTURES = {"amd64", "arm64v8", "ppc64le"} -CI_ARCHITECTURES = {"amd64", "arm64v8"} -SUPPORTED_OPERATING_SYSTEMS = {"centos-7"} - - -def test_manifest_exists(repository, tag) -> bool: - url = f"https://registry.hub.docker.com/v2/repositories/{repository}/tags/{tag}" - req = urllib.request.Request(url, headers={"Accept": "application/json"}) - status = None - try: - with urllib.request.urlopen(req) as res: - status = int(res.status) - except urllib.error.HTTPError as e: - status = int(e.code) - return status is not None and status >= 200 and status < 300 - - -@click.group() -def cli(): - pass - - -@click.command() -@click.option("-R", "--registry", default="docker.io") -@click.option("-r", "--repository", default="efabless/openlane-tools") -@click.option( - "-o", - "--os", - "operating_system", - required=True, - type=click.Choice(SUPPORTED_OPERATING_SYSTEMS), -) -@click.option( - "-m", - "--architecture", - required=True, - type=click.Choice(SUPPORTED_ARCHITECTURES), -) -@click.argument("tool") -def pull_if_doesnt_exist(registry, repository, operating_system, architecture, tool): - """ - Requires *actual* Docker. Podman won't cut it. - """ - - def get_tag_for(os, arch=None): - return ( - subprocess.check_output( - [ - "python3", - "../dependencies/tool.py", - tool, - f"--docker-tag-for-os={os}", - ] - + ([f"--docker-arch={arch}"] if arch is not None else []) - ) - .decode("utf8") - .rstrip() - ) - - image_tag = None - skip_manifest = None - if tool == "build-base": - image_tag = os.getenv("BUILD_BASE_TAG") - skip_manifest = True - elif tool == "run-base": - image_tag = os.getenv("RUN_BASE_TAG") - skip_manifest = True - else: - image_tag = get_tag_for(operating_system, architecture) - skip_manifest = False - - image = f"{repository}:{image_tag}" - images = ( - subprocess.check_output(["docker", "images", image]) - .decode("utf8") - .rstrip() - .split("\n")[1:] - ) - if len(images) >= 1: - print(f"[*] Found {image}.") - return - - print(f"[*] {image} not found, pulling...") - - if test_manifest_exists(repository, image_tag): - subprocess.call(["docker", "pull", image]) - print(f"[*] Pulled {image}.") - else: - if os.getenv("BUILD_IF_CANT_PULL") != "1": - print(f"[*] {image} not found in the repository.") - exit(os.EX_UNAVAILABLE) - else: - print(f"[*] {image} not found in the repository, building...") - env = os.environ.copy() - env["BUILD_ARCH"] = architecture - subprocess.check_call(["make", f"build-{tool}"], env=env) - print(f"Built {image}.") - - if os.getenv("BUILD_IF_CANT_PULL_THEN_PUSH") != "1": - return - - # Not needed for buildx, but won't hurt - print(f"[*] Pushing {image} to the container repository...") - subprocess.check_call(["docker", "push", image]) - print(f"[*] Pushed {image}.") - - if skip_manifest: - return - - manifest_tag = get_tag_for(operating_system) - manifest_name = f"{repository}:{manifest_tag}" - - print(f"[*] Trying to create multi-arch manifest {manifest_name}...") - arch_images = [] - for arch in CI_ARCHITECTURES: - print(f"[*] Verifying if the image for {arch} has been pushed...") - arch_image_tag = get_tag_for(operating_system, arch) - arch_image = f"{repository}:{arch_image_tag}" - if not test_manifest_exists(repository, arch_image_tag): - print(f"[*] {arch_image} not yet pushed. Aborting multi-arch manifest.") - exit(os.EX_OK) - arch_images.append(arch_image) - - print("[*] All images verified, creating and pushing manifest...") - - subprocess.call(["docker", "manifest", "rm", manifest_name]) - subprocess.check_call(["docker", "manifest", "create", manifest_name, *arch_images]) - subprocess.check_call( - [ - "docker", - "manifest", - "push", - manifest_name, - ] - ) - - print("[*] Done.") - - -cli.add_command(pull_if_doesnt_exist) - - -@click.command() -@click.option("-r", "--repository", required=True) -@click.option( - "-o", - "--os", - "operating_system", - required=True, - type=click.Choice(SUPPORTED_OPERATING_SYSTEMS), -) -@click.argument("tools", nargs=-1) -def process_dockerfile_tpl(repository, operating_system, tools): - image_tags = [ - ( - subprocess.check_output( - [ - "python3", - "../dependencies/tool.py", - f"--docker-tag-for-os={operating_system}", - tool, - ] - ) - .decode("utf8") - .rstrip() - ) - for tool in tools - ] - - image_names = [f"{repository}:{tag}" for tag in image_tags] - - from_lines = [ - f"FROM {name}-${{ARCH}} as container{i}" for i, name in enumerate(image_names) - ] - - copy_lines = [ - f"COPY --from=container{i} /build /build" for i, _ in enumerate(image_names) - ] - - template = open("./openlane/Dockerfile.tpl").read() - - parts = template.split("# ") - parts.insert(1, "\n".join(from_lines)) - - from_filled = "\n".join(parts) - - parts = from_filled.split("# ") - parts.insert(1, "\n".join(copy_lines)) - - final = "\n".join(parts) - print(final) - - -cli.add_command(process_dockerfile_tpl) - - -@click.command() -@click.option( - "--filter", default=".", help="regular expression to match submodule paths" -) -@click.argument("repository") -@click.argument("commit") -def fetch_submodules_from_tarballs(filter, repository, commit): - """ - Must be run from inside an extracted repository tarball. - - Given the repository's URL and commit, which are available, a table of the - git submodules with their repositories, commits and paths is constructed and - then promptly downloaded and extracted using only the GitHub APIs (and curl), - no git involved. - - This makes things much faster than having to clone an repo's entire history then - its submodule's entire history. - """ - - repository_path_info: urllib.parse.SplitResult = urllib.parse.urlsplit(repository) - - # 1. Get Commits Of Submodules - api_result = None - - try: - api_result = subprocess.check_output( - [ - "curl", - "--fail", - "-s", - "-L", - "-H", - "Accept: application/vnd.github.v3+json", - f"https://api.github.com/repos{repository_path_info.path}/git/trees/{commit}?recursive=True", - ] - ) - except Exception as e: - print(e, file=sys.stderr) - sys.exit(os.EX_DATAERR) - - api_result_parsed = json.loads(api_result) - api_result_tree = api_result_parsed["tree"] - submodules = [element for element in api_result_tree if element["type"] == "commit"] - shas_by_path = {submodule["path"]: submodule["sha"] for submodule in submodules} - - # 2. Get Submodule Manifest - api_result = None - - try: - api_result = subprocess.check_output( - [ - "curl", - "--fail", - "-s", - "-L", - f"https://raw.githubusercontent.com/{repository_path_info.path}/{commit}/.gitmodules", - ] - ) - except Exception as e: - print(e, file=sys.stderr) - sys.exit(os.EX_DATAERR) - - gitmodules = api_result.decode("utf8") - - section_line_rx = re.compile(r"\[\s*submodule\s+\"([\w\-\.\/]+)\"\]") - key_value_line_rx = re.compile(r"(\w+)\s*=\s*(.+)") - - submodules_by_name = {} - current = {} # First one is discarded - for line in gitmodules.split("\n"): - section_match = section_line_rx.search(line) - if section_match is not None: - name = section_match[1] - submodules_by_name[name] = {} - current = submodules_by_name[name] - - kvl_match = key_value_line_rx.search(line) - if kvl_match is not None: - key, value = kvl_match[1], kvl_match[2] - current[key] = value - - for name, submodule in submodules_by_name.items(): - submodule["commit"] = shas_by_path.get(submodule["path"]) - if submodule["url"].startswith(tuple(["./", "../"])): - submodule["url"] = urllib.parse.urljoin(repository, submodule["url"]) - - if submodule["url"].endswith(".git"): - submodule["url"] = submodule["url"][:-4] - - # 3. Extract Submodules - temp_dir = tempfile.gettempdir() - filter_rx = re.compile(filter, flags=re.I) - for (name, values) in submodules_by_name.items(): - path = values["path"] - - if filter_rx.match(path) is None: - print(f"Skipping {path}...", flush=True) - continue - else: - print(f"Expanding {path}...", flush=True) - - name_fs = re.sub(r"\/", "_", name) - tarball = os.path.join(temp_dir, f"{name_fs}.tar.gz") - - url = values["url"] - commit = values["commit"] - - url = os.path.join(url, "tarball", commit) - - print(f"Downloading {url} to {path}...", file=sys.stderr) - subprocess.check_call(["curl", "-sL", "-o", tarball, url]) - - shutil.rmtree(path, ignore_errors=True) - - pathlib.Path(path).mkdir(parents=True, exist_ok=True) - - subprocess.check_call( - ["tar", "-xzf", tarball, "--strip-components=1", "-C", path] - ) - - -cli.add_command(fetch_submodules_from_tarballs) - - -@click.command() -@click.option( - "-i", - "--ignoring", - type=str, - multiple=True, - help="Ignore paths `fnmatch`-ing one or more of these filters.", -) -@click.argument("sources", nargs=-1) -@click.argument("destination", nargs=1) -def copy_tree(ignoring, sources, destination): - def ignore_runs(src, entries): - result = set() - for entry in entries: - entry_path = os.path.join(src, entry) - for ignored in ignoring: - if fnmatch.fnmatch(entry_path, ignored): - result.add(entry) - return result - - for source in list(sources): - if os.path.isfile(source): - shutil.copy(source, destination) - elif os.path.isdir(source): - shutil.copytree(source, destination, ignore=ignore_runs) - else: - raise FileNotFoundError(source) - - -cli.add_command(copy_tree) - - -if __name__ == "__main__": - cli() diff --git a/docker/verilator/Dockerfile b/docker/verilator/Dockerfile deleted file mode 100644 index 5ffa8833e..000000000 --- a/docker/verilator/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2020 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG RUN_BASE_IMAGE= -ARG BUILD_BASE_IMAGE= -FROM ${BUILD_BASE_IMAGE} AS builder - -ARG VERILATOR_REPO -ARG VERILATOR_COMMIT - -WORKDIR /verilator -RUN curl -L ${VERILATOR_REPO}/tarball/${VERILATOR_COMMIT} | tar -xzC . --strip-components=1 && \ - autoconf &&\ - ./configure --prefix=/build &&\ - make -j$(nproc)&&\ - make install - -RUN rm /build/share/verilator/include/verilated_std.sv &&\ - touch /build/share/verilator/include/verilated_std.sv - -RUN mkdir -p /build/version -RUN date +"Build Timestamp: %Y-%m-%d_%H-%M-%S" > /build/version/verilator.version -RUN echo ${VERILATOR_COMMIT} >> /build/version/verilator.version -RUN tar -czf /build.tar.gz /build - -# --- -FROM ${RUN_BASE_IMAGE} AS runnable -ENV PATH /build/bin:$PATH - -COPY --from=builder /build /build -COPY --from=builder /build.tar.gz /build.tar.gz diff --git a/docker/vlogtoverilog/Dockerfile b/docker/vlogtoverilog/Dockerfile deleted file mode 100644 index eca1e4ff3..000000000 --- a/docker/vlogtoverilog/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2020-2021 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ARG RUN_BASE_IMAGE= -ARG BUILD_BASE_IMAGE= -FROM ${BUILD_BASE_IMAGE} AS builder - -ARG VLOGTOVERILOG_REPO -ARG VLOGTOVERILOG_COMMIT - -WORKDIR /qflow -RUN curl -L ${VLOGTOVERILOG_REPO}/tarball/${VLOGTOVERILOG_COMMIT} | tar -xzC . --strip-components=1 && \ - ./configure && \ - cd src && \ - make vlog2Verilog - -RUN mkdir -p /build/bin && cp /qflow/src/vlog2Verilog /build/bin/ - -RUN mkdir -p /build/version/ -RUN date +"Build Timestamp: %Y-%m-%d_%H-%M-%S" > /build/version/vlog2Verilog.version -RUN echo ${VLOGTOVERILOG_COMMIT} >> /build/version/vlog2Verilog.version -RUN tar -czf /build.tar.gz /build - -# --- -FROM ${RUN_BASE_IMAGE} AS runnable -ENV PATH /build/bin:$PATH - -COPY --from=builder /build /build -COPY --from=builder /build.tar.gz /build.tar.gz diff --git a/docker/yosys/Dockerfile b/docker/yosys/Dockerfile deleted file mode 100644 index c5f76c62a..000000000 --- a/docker/yosys/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2020 Efabless Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# syntax = docker/dockerfile:1.0-experimental -ARG RUN_BASE_IMAGE= -ARG BUILD_BASE_IMAGE= -FROM ${BUILD_BASE_IMAGE} AS builder - -# git clone yosys -ARG YOSYS_REPO -ARG YOSYS_COMMIT - -WORKDIR /yosys -RUN curl -L ${YOSYS_REPO}/tarball/${YOSYS_COMMIT} | tar -xzC . --strip-components=1 && \ - make PREFIX=/build config-gcc &&\ - make PREFIX=/build -j$(nproc) &&\ - make PREFIX=/build install - - -RUN mkdir -p /build/version -RUN date +"Build Timestamp: %Y-%m-%d_%H-%M-%S" > /build/version/yosys.version -RUN echo ${YOSYS_COMMIT} >> /build/version/yosys.version -RUN tar -czf /build.tar.gz /build - -# --- -FROM ${RUN_BASE_IMAGE} AS runnable -ENV PATH /build/bin:$PATH - -COPY --from=builder /build /build -COPY --from=builder /build.tar.gz /build.tar.gz diff --git a/docs/source/for_developers/gha_workflow.md b/docs/source/for_developers/gha_workflow.md index 3cf7ceaee..24e0e2c51 100644 --- a/docs/source/for_developers/gha_workflow.md +++ b/docs/source/for_developers/gha_workflow.md @@ -7,6 +7,7 @@ The deployment flow occurs on a daily basis. The PR flow happens whenever someon ![A Diagram Of The Flow](../../_static/gha.png) * A maintainer cannot review their own code, but they can merge it after a review by another maintainer. +* The tool updater is no longer in use and has been removed. ## Required Secrets/Variables Repository secrets are used to protect certain credentials, while variables are repository-dependent parameters for the CI. @@ -31,10 +32,3 @@ Repository secrets are used to protect certain credentials, while variables are |---------------|---------------------------------------------------------------| | `DOCKERHUB_USER` | A username for a user that has push access to the organization that owns `DOCKER_IMAGE` on Docker Hub. In our case, that's an Efabless Employee with push access. | | `DOCKERHUB_PASSWORD` | The password/token for the given username that has push access to the organization that owns `DOCKER_IMAGE` on Docker Hub. | - -### Tool Updater - -| Variable | Description | -|---------------|---------------------------------------------------------------| -| `FORK_NAME` | A fork to push branches for tool updates to. Format `bot-account/OpenLane` | -| `BOT_AUTHOR_LINE` | A git author line for the bot account, i.e. `Firstname Lastname `. | \ No newline at end of file diff --git a/docs/source/for_developers/tool_versions.md b/docs/source/for_developers/tool_versions.md new file mode 100644 index 000000000..9b997bde9 --- /dev/null +++ b/docs/source/for_developers/tool_versions.md @@ -0,0 +1,49 @@ +# Updating Tool Versions + +As a rule, tool versions match those of +[OpenLane 2](https://github.com/efabless/openlane2). A certain commit of +OpenLane 2 is pinned in `flake.lock`. The OpenLane 2 repo lists the tools under +`nix/`, where the commits for all tools are pinned within the requisite +derivations. + +The pin of OpenLane 2 may be updated by running `nix flake lock --update-input` +at the root of the repo. This will set it to the latest commit in main. + + diff --git a/docs/source/getting_started/installation/index.rst b/docs/source/getting_started/installation/index.rst index a9e8f4285..28e84e11e 100644 --- a/docs/source/getting_started/installation/index.rst +++ b/docs/source/getting_started/installation/index.rst @@ -8,4 +8,3 @@ Installation installation_macos installation_win installation_linux - installation_local \ No newline at end of file diff --git a/docs/source/getting_started/installation/installation_local.md b/docs/source/getting_started/installation/installation_local.md deleted file mode 100644 index 831188df4..000000000 --- a/docs/source/getting_started/installation/installation_local.md +++ /dev/null @@ -1,32 +0,0 @@ -# Containerless/Local - -:::{warning} -The local installer is no longer actively supported. Unless you ***absolutely*** know what you're doing, please use the Docker image. -::: - -At its core, OpenLane is a set of scripts working with a set of tools. If you'd like to avoid using a Docker container, you can, but you will have to set up all of the tools required by OpenLane on your computer. We do provide a best-effort script to assist you with that. - -## Base Requirements -* Python 3.6+ - - pip - - venv - - pyyaml (`python3 -m pip install pyyaml`) - -## Tool Library -You can run `python3 ./env.py tool-list` for a list. There are at least a dozen tools to install here. Luckily, you don't have to install them all one-by-one: There is an installation script that installs most of them. - -You can invoke `python3 ./env.py local-install`. This tool copies the skeleton and installs all the tools to `$OPENLANE_ROOT_DIR/install`. Furthermore, if you are on CentOS 7, macOS, Ubuntu 20.04 or Arch Linux, the installer will offer to install all the required apt, yum or brew packages for you. - -The tools will all be installed with `./install` as a prefix. You'll find all the repos in `./install/build/repos` and a list of versions in `./install/build/versions`. - -**DO NOTE:** We expect you to get some tools on your own, because said tools are too complex to build in an automated fashion. Namely: -* OpenROAD -* KLayout -* Git 2.34+ - -After the installer is done, you can simply invoke `./flow.tcl` outside of Docker and it should work okay. - -## How this works -`flow.tcl` looks for a file called `./install/env.tcl` before it does anything. If it finds it, it sources it. The `./install` directory is aliased in Docker environments, which already have the proper tools installed. - -`./install/env.tcl` contains the necessary environment variables to add the installed tools to PATH and activate the Python virtual environment. diff --git a/docs/source/getting_started/quickstart.md b/docs/source/getting_started/quickstart.md index 6cbc640fb..b45ff7e67 100644 --- a/docs/source/getting_started/quickstart.md +++ b/docs/source/getting_started/quickstart.md @@ -10,10 +10,6 @@ This guide covers running the flow on existing desings, adding new designs and q ## Starting the OpenLane Environment -:::{note} -If you installed OpenLane following [local installation](installation/installation_local.md) steps, these instructions will not entirely apply. We no longer actively support local installation. -::: - OpenLane uses Docker to create a reproducible environment for your projects. You don't need any extra steps to run the Docker image, as the Makefile already takes care of it. Just run the following commands to enter the OpenLane environment: ```sh diff --git a/env.py b/env.py index bcd35de7c..f1cd1000a 100755 --- a/env.py +++ b/env.py @@ -45,13 +45,6 @@ def tool_list(): print("%s %s" % (tool.name, tool.version_string)) -def local_install(): - from dependencies.installer import Installer - - installer = Installer() - installer.run() - - def docker_config(): from dependencies.env_info import ContainerInfo @@ -232,7 +225,6 @@ def main(): args = sys.argv[1:] commands = { "tool-list": tool_list, - "local-install": local_install, "docker-config": docker_config, "issue-survey": issue_survey, } diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..507b273d3 --- /dev/null +++ b/flake.lock @@ -0,0 +1,123 @@ +{ + "nodes": { + "flake-compat": { + "locked": { + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "revCount": 57, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.0.1/018afb31-abd1-7bff-a5e4-cff7e18efb7a/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz" + } + }, + "ioplace-parser": { + "inputs": { + "nixpkgs": [ + "openlane2", + "nixpkgs" + ] + }, + "locked": { + "narHash": "sha256-0Lt3DVImH3TpwUh7sDW/6Cxsrmo5DDG+SCuirBFquXs=", + "owner": "efabless", + "repo": "ioplace_parser", + "rev": "f1c163e8184fbce2676a19a1d28c3e6c0b5ddaf2", + "type": "github" + }, + "original": { + "owner": "efabless", + "repo": "ioplace_parser", + "type": "github" + } + }, + "libparse": { + "inputs": { + "nixpkgs": [ + "openlane2", + "nixpkgs" + ] + }, + "locked": { + "narHash": "sha256-1w6HBBE2bWAD0GM98O8WZRmZDW9+EzD0KFvnnH2ho/k=", + "owner": "efabless", + "repo": "libparse-python", + "rev": "cec8b6dfd3d1c97bc5ccd1d0a41c44e51bc9c1eb", + "type": "github" + }, + "original": { + "owner": "efabless", + "repo": "libparse-python", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "narHash": "sha256-C36QmoJd5tdQ5R9MC1jM7fBkZW9zBUqbUCsgwS6j4QU=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "c1be43e8e837b8dbee2b3665a007e761680f0c3d", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "openlane2": { + "inputs": { + "flake-compat": "flake-compat", + "ioplace-parser": "ioplace-parser", + "libparse": "libparse", + "nixpkgs": "nixpkgs", + "volare": "volare" + }, + "locked": { + "lastModified": 1715082706, + "narHash": "sha256-OUaDO8RtK76NN3yuJZIwtel4kuRVmALkXJnTdMIeT8U=", + "owner": "efabless", + "repo": "openlane2", + "rev": "058a93dfacf5352abd06931c3e45fec08cfa468e", + "type": "github" + }, + "original": { + "owner": "efabless", + "ref": "dev", + "repo": "openlane2", + "type": "github" + } + }, + "root": { + "inputs": { + "openlane2": "openlane2" + } + }, + "volare": { + "inputs": { + "nixpkgs": [ + "openlane2", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1715078431, + "narHash": "sha256-w4NLYQwvE/UFgPZDOnKDpagTjKb8FwKWOp3wV/RvPFM=", + "owner": "efabless", + "repo": "volare", + "rev": "13ccef8de5d8b27311ffd458ac629953784b5224", + "type": "github" + }, + "original": { + "owner": "efabless", + "repo": "volare", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..019b71e21 --- /dev/null +++ b/flake.nix @@ -0,0 +1,72 @@ +# Copyright 2024 Efabless Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +{ + nixConfig = { + extra-substituters = [ + "https://openlane.cachix.org" + ]; + extra-trusted-public-keys = [ + "openlane.cachix.org-1:qqdwh+QMNGmZAuyeQJTH9ErW57OWSvdtuwfBKdS254E=" + ]; + }; + + inputs = { + openlane2.url = github:efabless/openlane2/dev; + }; + + outputs = { + self, + openlane2, + ... + }: let + nixpkgs = openlane2.inputs.nixpkgs; + in { + # Helper functions + forAllSystems = function: + nixpkgs.lib.genAttrs [ + "x86_64-linux" + "aarch64-linux" + "x86_64-darwin" + "aarch64-darwin" + ] ( + system: + function (import nixpkgs { + inherit system; + overlays = []; + }) + ); + + # Outputs + packages = self.forAllSystems (pkgs: let + callPackage = pkgs.lib.callPackageWith (pkgs // openlane2.packages."${pkgs.system}" // self.packages.${pkgs.system} ); + callPythonPackage = pkgs.lib.callPackageWith (pkgs // pkgs.python3.pkgs // openlane2.packages."${pkgs.system}" // openlane2.inputs.libparse.packages."${pkgs.system}" // openlane2.inputs.volare.packages."${pkgs.system}" // self.packages.${pkgs.system}); + in + rec { + # ADD OVERRIDES HERE + openlane1 = callPythonPackage ./default.nix {}; + default = openlane1; + } + // (pkgs.lib.optionalAttrs (pkgs.stdenv.isLinux) {openlane1-docker = callPackage ./docker/docker.nix { + createDockerImage = openlane2.createDockerImage; + };})); + + # devShells = self.forAllSystems ( + # pkgs: let + # callPackage = pkgs.lib.callPackageWith (pkgs // self.packages.${pkgs.system}); + # callPythonPackage = pkgs.lib.callPackageWith (pkgs // pkgs.python3.pkgs // self.packages.${pkgs.system}); + # in rec { + # } + # ); + }; +} diff --git a/flow.tcl b/flow.tcl index 710f19adf..645b26607 100755 --- a/flow.tcl +++ b/flow.tcl @@ -14,10 +14,6 @@ # limitations under the License. set ::env(OPENLANE_ROOT) [file dirname [file normalize [info script]]] -if { [file exists $::env(OPENLANE_ROOT)/install/env.tcl ] } { - source $::env(OPENLANE_ROOT)/install/env.tcl -} - if { ! [info exists ::env(OPENROAD_BIN) ] } { set ::env(OPENROAD_BIN) openroad } @@ -375,11 +371,9 @@ set flags {-interactive -it -drc -lvs -synth_explore -run_hooks} parse_key_args "flow.tcl" argv arg_values $options flags_map $flags -no_consume -if {[catch {exec cat $::env(OPENLANE_ROOT)/install/installed_version} ::env(OPENLANE_VERSION)]} { - if {[catch {exec cat /git_version} ::env(OPENLANE_VERSION)]} { - if {[catch {exec git --git-dir $::env(OPENLANE_ROOT)/.git rev-parse HEAD} ::env(OPENLANE_VERSION)]} { - set ::env(OPENLANE_VERSION) "UNKNOWN" - } +if {[catch {exec cat /git_version} ::env(OPENLANE_VERSION)]} { + if {[catch {exec git --git-dir $::env(OPENLANE_ROOT)/.git rev-parse HEAD} ::env(OPENLANE_VERSION)]} { + set ::env(OPENLANE_VERSION) "" } } diff --git a/gui.py b/gui.py index 375ade079..26fdaee8d 100755 --- a/gui.py +++ b/gui.py @@ -16,6 +16,7 @@ import click import os +import shlex import subprocess import glob @@ -94,10 +95,9 @@ def gui(viewer, format, run_dir, stage): else: layout = matches[0] - subprocess.check_call( + run_env = os.environ.copy() + run_env["KLAYOUT_ARGV"] = shlex.join( [ - "python3", - "./scripts/klayout/open_design.py", "--input-lef", run_config["MERGED_LEF"], "--lyt", @@ -109,6 +109,14 @@ def gui(viewer, format, run_dir, stage): layout, ] ) + subprocess.check_call( + [ + "klayout", + "-rm", + "./scripts/klayout/open_design.py", + ], + env=run_env, + ) if __name__ == "__main__": diff --git a/scripts/klayout/open_design.py b/scripts/klayout/open_design.py index dce116770..aa5c3256a 100755 --- a/scripts/klayout/open_design.py +++ b/scripts/klayout/open_design.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Copyright 2020-2022 Efabless Corporation # Copyright 2021 The American University in Cairo and the Cloud V Project # @@ -13,93 +12,62 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import os import sys -from typing import TYPE_CHECKING - -if "klayout" in os.path.basename(sys.executable): - import pya -else: - import click - - @click.command() - @click.option( - "-l", - "--input-lef", - "input_lefs", - multiple=True, - ) - @click.option( - "-T", - "--lyt", - required=True, - help="KLayout .lyt file", - ) - @click.option( - "-P", - "--lyp", - required=True, - help="KLayout .lyp file", - ) - @click.option( - "-M", - "--lym", - required=True, - help="KLayout .map (LEF/DEF layer map) file", - ) - @click.argument("input") - def cli(**kwargs): - args = [ - "klayout", - "-rm", - __file__, - ] - for key, value in kwargs.items(): - args.append("-rd") - if isinstance(value, tuple) or isinstance(value, list): - value = ";".join(value) - elif ( - isinstance(value, str) - and os.path.exists(value) - and key != "design_name" - ): - value = os.path.abspath(value) - - args.append(f"{key}={value or 'NULL'}") - - os.execlp("klayout", *args) - - if __name__ == "__main__": - cli() +import shlex +from typing import Tuple +import pya # Must be run inside KLayout-- the library version of pya does not include "Application" +import click -if TYPE_CHECKING: - # Dummy data for type-checking - input: str = "" - input_lefs: str = "" - lyp: str = "" - lyt: str = "" - lym: str = "" -try: - main_window = pya.Application.instance().main_window() +@click.command() +@click.option( + "-l", + "--input-lef", + "input_lefs", + multiple=True, +) +@click.option( + "-T", + "--lyt", + required=True, + help="KLayout .lyt file", +) +@click.option( + "-P", + "--lyp", + required=True, + help="KLayout .lyp file", +) +@click.option( + "-M", + "--lym", + required=True, + help="KLayout .map (LEF/DEF layer map) file", +) +@click.argument("input") +def open_design(input_lefs: Tuple[str, ...], lyt: str, lyp: str, lym: str, input: str): + try: + main_window = pya.Application.instance().main_window() - tech = pya.Technology() - tech.load(lyt) + tech = pya.Technology() + tech.load(lyt) - layout_options = tech.load_layout_options - layout_options.keep_other_cells = True - layout_options.lefdef_config.macro_resolution_mode = 1 - layout_options.lefdef_config.read_lef_with_def = False - layout_options.lefdef_config.lef_files = input_lefs.split(";") - layout_options.lefdef_config.map_file = lym + layout_options = tech.load_layout_options + layout_options.keep_other_cells = True + layout_options.lefdef_config.macro_resolution_mode = 1 + layout_options.lefdef_config.read_lef_with_def = False + layout_options.lefdef_config.lef_files = list(input_lefs) + layout_options.lefdef_config.map_file = lym - cell_view = main_window.load_layout(input, layout_options, 0) - layout_view = cell_view.view() - layout_view.load_layer_props(lyp) + cell_view = main_window.load_layout(input, layout_options, 0) + layout_view = cell_view.view() + layout_view.load_layer_props(lyp) + except Exception as e: + print(e, file=sys.stderr) + pya.Application.instance().exit(1) -except Exception as e: - print(e, file=sys.stderr) - pya.Application.instance().exit(1) +if __name__ == "__main__": + open_design(shlex.split(os.getenv("KLAYOUT_ARGV"))) diff --git a/scripts/klayout/stream_out.py b/scripts/klayout/stream_out.py index 2242cfd15..3de4241b5 100755 --- a/scripts/klayout/stream_out.py +++ b/scripts/klayout/stream_out.py @@ -44,162 +44,129 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -import os -import sys -from typing import TYPE_CHECKING, Optional - -if "klayout" in os.path.basename(sys.executable): - import pya -else: - import click - - @click.command() - @click.option("-o", "--output", required=True) - @click.option( - "-l", - "--input-lef", - "input_lefs", - multiple=True, - ) - @click.option( - "-T", - "--lyt", - required=True, - help="KLayout .lyt file", - ) - @click.option( - "-P", - "--lyp", - required=True, - help="KLayout .lyp file", - ) - @click.option( - "-M", - "--lym", - required=True, - help="KLayout .map (LEF/DEF layer map) file", - ) - @click.option("-w", "--with-gds-file", "input_gds_files", multiple=True, default=[]) - @click.option("-s", "--seal-gds-file", "seal_gds", default=None) - @click.option( - "-t", - "--top", - "design_name", - required=True, - help="Name of the design/top module", - ) - @click.argument("input") - def stream_out(**kwargs): - args = [ - "klayout", - "-b", - "-rm", - __file__, - ] - for key, value in kwargs.items(): - args.append("-rd") - if isinstance(value, tuple) or isinstance(value, list): - value = ";".join(value) - elif ( - isinstance(value, str) - and os.path.exists(value) - and key != "design_name" - ): - value = os.path.abspath(value) - - args.append(f"{key}={value or 'NULL'}") - - os.execlp("klayout", *args) - - if __name__ == "__main__": - stream_out() - - -if TYPE_CHECKING: - # Dummy data for type-checking - input: str = "" - output: str = "" - input_lefs: str = "" - lyp: str = "" - lyt: str = "" - lym: str = "" - design_name: str = "" - input_gds_files: str = "" - seal_gds: Optional[str] = "" - -if seal_gds == "NULL": - seal_gds = None - - -try: - # Load technology file - tech = pya.Technology() - tech.load(lyt) - layout_options = tech.load_layout_options - layout_options.lefdef_config.macro_resolution_mode = 1 - layout_options.lefdef_config.read_lef_with_def = False - layout_options.lefdef_config.lef_files = input_lefs.split(";") - layout_options.lefdef_config.map_file = lym - - # Load def file - main_layout = pya.Layout() - # main_layout.load_layer_props(props_file) - main_layout.read(input, layout_options) - - # Clear cells - top_cell_index = main_layout.cell(design_name).cell_index() - - print("[INFO] Clearing cells…") - for i in main_layout.each_cell(): - if i.cell_index() != top_cell_index: - if not i.name.startswith("VIA"): - i.clear() - - # Load in the gds to merge - print("[INFO] Merging GDS files…") - for gds in input_gds_files.split(";"): - print(f"\t{gds}") - main_layout.read(gds) - - # Copy the top level only to a new layout - print(f"[INFO] Copying top level cell '{design_name}'…") - top_only_layout = pya.Layout() - top_only_layout.dbu = main_layout.dbu - top = top_only_layout.create_cell(design_name) - top.copy_tree(main_layout.cell(design_name)) - - print("[INFO] Checking for missing GDS…") - missing_gds = False - for i in top_only_layout.each_cell(): - if i.is_empty(): - missing_gds = True - print(f"[ERROR] LEF Cell '{i.name}' has no matching GDS cell.") - - if missing_gds: - raise Exception("One or more cell GDS files are missing.") - else: - print("[INFO] All LEF cells have matching GDS cells.") - - if seal_gds is not None: - top_cell = top_only_layout.top_cell() - - print(f"[INFO] Reading seal GDS file '{seal_gds}'…") - top_only_layout.read(seal_gds) - - for cell in top_only_layout.top_cells(): - if cell != top_cell: - print(f"[INFO] Merging '{cell.name}' as child of '{top_cell.name}'…") - top.insert(pya.CellInstArray(cell.cell_index(), pya.Trans())) - - # Write out the GDS - print(f"[INFO] Writing out GDS '{output}'…") - top_only_layout.write(output) - print("[INFO] Done.") - - pya.Application.instance().exit(0) -except Exception as e: - print(e) - - pya.Application.instance().exit(1) +from typing import Tuple, Optional + +import pya +import click + + +@click.command() +@click.option("-o", "--output", required=True) +@click.option( + "-l", + "--input-lef", + "input_lefs", + multiple=True, +) +@click.option( + "-T", + "--lyt", + required=True, + help="KLayout .lyt file", +) +@click.option( + "-P", + "--lyp", + required=True, + help="KLayout .lyp file", +) +@click.option( + "-M", + "--lym", + required=True, + help="KLayout .map (LEF/DEF layer map) file", +) +@click.option("-w", "--with-gds-file", "input_gds_files", multiple=True, default=[]) +@click.option("-s", "--seal-gds-file", "seal_gds", default=None) +@click.option( + "-t", + "--top", + "design_name", + required=True, + help="Name of the design/top module", +) +@click.argument( + "input", + type=click.Path(exists=True, file_okay=True, dir_okay=False), +) +def stream_out( + output: str, + input_lefs: Tuple[str, ...], + lyt: str, + lyp: str, + lym: str, + input_gds_files: Tuple[str, ...], + seal_gds: Optional[str], + design_name: str, + input: str, +): # Load technology file + try: + tech = pya.Technology() + tech.load(lyt) + layout_options = tech.load_layout_options + layout_options.lefdef_config.read_lef_with_def = False + layout_options.lefdef_config.lef_files = list(input_lefs) + layout_options.lefdef_config.map_file = lym + + # Load def file + main_layout = pya.Layout() + # main_layout.load_layer_props(props_file) + main_layout.read(input, layout_options) + + # Clear cells + top_cell_index = main_layout.cell(design_name).cell_index() + + print("[INFO] Clearing cells…") + for i in main_layout.each_cell(): + if i.cell_index() != top_cell_index: + if not i.name.startswith("VIA"): + i.clear() + + # Load in the gds to merge + print("[INFO] Merging GDS files…") + for gds in input_gds_files: + main_layout.read(gds) + + # Copy the top level only to a new layout + print(f"[INFO] Copying top level cell '{design_name}'…") + top_only_layout = pya.Layout() + top_only_layout.dbu = main_layout.dbu + top = top_only_layout.create_cell(design_name) + top.copy_tree(main_layout.cell(design_name)) + + print("[INFO] Checking for missing GDS…") + missing_gds = False + for i in top_only_layout.each_cell(): + if i.is_empty(): + missing_gds = True + print(f"[ERROR] LEF Cell '{i.name}' has no matching GDS cell.") + + if missing_gds: + raise Exception("One or more cell GDS files are missing.") + else: + print("[INFO] All LEF cells have matching GDS cells.") + + if seal_gds is not None: + top_cell = top_only_layout.top_cell() + + print(f"[INFO] Reading seal GDS file '{seal_gds}'…") + top_only_layout.read(seal_gds) + + for cell in top_only_layout.top_cells(): + if cell != top_cell: + print( + f"[INFO] Merging '{cell.name}' as child of '{top_cell.name}'…" + ) + top.insert(pya.CellInstArray(cell.cell_index(), pya.Trans())) + + # Write out the GDS + print(f"[INFO] Writing out GDS '{output}'…") + top_only_layout.write(output) + print("[INFO] Done.") + except Exception as e: + print(e) + exit(1) + + +if __name__ == "__main__": + stream_out() diff --git a/scripts/odbpy/reader.py b/scripts/odbpy/reader.py index 19fd0aeee..aec97eaaa 100644 --- a/scripts/odbpy/reader.py +++ b/scripts/odbpy/reader.py @@ -11,32 +11,29 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import odb +from openroad import Tech, Design +from typing import ClassVar, Optional + import os import sys import inspect import functools -from typing import Optional import click -import odb -from utl import Logger -from openroad import Tech, Design - class OdbReader(object): - logger: Optional[Logger] = None + primary_reader: ClassVar[Optional["OdbReader"]] = None def __init__(self, *args): - if self.__class__.logger is None: - ord_tech = Tech() - design = Design(ord_tech) - - self.db = ord_tech.getDB() - self.__class__.logger = design.getLogger() - else: + if primary := OdbReader.primary_reader: self.db = odb.dbDatabase.create() - self.db.setLogger(self.__class__.logger) + self.db.setLogger(primary.design.getLogger()) + else: + self.ord_tech = Tech() + self.design = Design(self.ord_tech) + self.db = self.ord_tech.getDB() if len(args) == 1: db_in = args[0] @@ -59,6 +56,9 @@ def __init__(self, *args): self.dbunits = self.block.getDefUnits() self.instances = self.block.getInsts() + if OdbReader.primary_reader is None: + OdbReader.primary_reader = self + def add_lef(self, new_lef): odb.read_lef(self.db, new_lef) diff --git a/scripts/tcl_commands/all.tcl b/scripts/tcl_commands/all.tcl index a8bec1f90..d697ac5d0 100755 --- a/scripts/tcl_commands/all.tcl +++ b/scripts/tcl_commands/all.tcl @@ -130,8 +130,7 @@ proc prep_lefs {args} { if { [info exists ::env(METAL_LAYER_NAMES)] } { set ::env(TECH_METAL_LAYERS) $::env(METAL_LAYER_NAMES) } else { - try_exec $::env(OPENROAD_BIN) -exit -no_init -python\ - $::env(SCRIPTS_DIR)/odbpy/lefutil.py get_metal_layers\ + run_odbpy_script $::env(SCRIPTS_DIR)/odbpy/lefutil.py get_metal_layers\ -o $::env(TMP_DIR)/layers.list\ $arg_values(-tech_lef) @@ -1313,8 +1312,7 @@ proc save_final_views {args} { proc run_post_run_hooks {} { if { [file exists $::env(DESIGN_DIR)/hooks/post_run.py]} { puts_info "Running post run hook..." - set result [exec $::env(OPENROAD_BIN) -exit -no_init -python $::env(DESIGN_DIR)/hooks/post_run.py] - puts_info "$result" + run_odbpy_script $::env(DESIGN_DIR)/hooks/post_run.py } else { puts_verbose "No post-run hook found, skipping..." } diff --git a/scripts/tcl_commands/floorplan.tcl b/scripts/tcl_commands/floorplan.tcl index 519523882..802f0c902 100755 --- a/scripts/tcl_commands/floorplan.tcl +++ b/scripts/tcl_commands/floorplan.tcl @@ -19,7 +19,7 @@ proc extract_core_dims {args} { set out_tmp $::env(TMP_DIR)/dimensions.txt - try_exec $::env(OPENROAD_BIN) -exit -no_init -python $::env(SCRIPTS_DIR)/odbpy/defutil.py extract_core_dims\ + run_odbpy_script $::env(SCRIPTS_DIR)/odbpy/defutil.py extract_core_dims\ --output-data $out_tmp\ --input-lef $::env(MERGED_LEF)\ $::env(CURRENT_DEF) @@ -75,7 +75,7 @@ proc init_floorplan {args} { set intermediate [index_file $::env(floorplan_tmpfiles)/minimized_pdn.txt] - try_exec $::env(OPENROAD_BIN) -exit -no_init -python $::env(SCRIPTS_DIR)/odbpy/snap_to_grid.py\ + run_odbpy_script $::env(SCRIPTS_DIR)/odbpy/snap_to_grid.py\ --output $intermediate\ --input-lef $::env(MERGED_LEF)\ [expr {$core_width/8.0}] [expr {$core_height/8.0}] [expr {$core_width/4.0}] [expr {$core_height/4.0}] @@ -440,7 +440,7 @@ proc padframe_gen_batch {args} { } puts_info "Generating pad frame" - try_exec openroad -python -exit $::env(SCRIPTS_DIR)/odbpy/padringer.py\ + run_odbpy_script $::env(SCRIPTS_DIR)/odbpy/padringer.py\ --def-netlist $arg_values(-def) \ {*}$prefix_argument \ {*}$lefs_argument \ diff --git a/scripts/tcl_commands/lvs.tcl b/scripts/tcl_commands/lvs.tcl index ca1d52631..0d45218b9 100755 --- a/scripts/tcl_commands/lvs.tcl +++ b/scripts/tcl_commands/lvs.tcl @@ -93,7 +93,7 @@ proc write_powered_verilog {args} { set_if_unset arg_values(-powered_netlist) "" } - try_exec $::env(OPENROAD_BIN) -exit -no_init -python $::env(SCRIPTS_DIR)/odbpy/power_utils.py write_powered_def\ + run_odbpy_script $::env(SCRIPTS_DIR)/odbpy/power_utils.py write_powered_def\ --output $arg_values(-output_def) \ --input-lef $arg_values(-lef) \ --power-port $arg_values(-power) \ diff --git a/scripts/utils/utils.tcl b/scripts/utils/utils.tcl index a583a2742..66e1a631b 100755 --- a/scripts/utils/utils.tcl +++ b/scripts/utils/utils.tcl @@ -514,7 +514,7 @@ proc manipulate_layout {args} { set_if_unset arg_values(-output) $arg_values(-input) set_if_unset arg_values(-output_def) /dev/null - try_exec $::env(OPENROAD_BIN) -exit -no_init -python\ + run_odbpy_script\ {*}$args \ --input-lef $::env(MERGED_LEF) \ --output-def $arg_values(-output_def) \ @@ -780,4 +780,10 @@ proc run_tcl_script {args} { } } +proc run_odbpy_script {args} { + set ::env(PYTHONPATH) [exec python3 -c "import sys; import site; print(':'.join(site.getsitepackages() + sys.path), end='')"] + try_exec $::env(OPENROAD_BIN) -exit -no_init -python {*}$args + unset ::env(PYTHONPATH) +} + package provide openlane_utils 0.9 diff --git a/tests/1413-odb_remover/interactive.tcl b/tests/1413-odb_remover/interactive.tcl index 7b1d98018..6b93991e7 100644 --- a/tests/1413-odb_remover/interactive.tcl +++ b/tests/1413-odb_remover/interactive.tcl @@ -6,19 +6,19 @@ try_catch echo { read_lef $::env(MERGED_LEF) read_def $::env(DESIGN_DIR)/in.def write_db $::env(DESIGN_DIR)/in.odb - } | openroad -exit +} | openroad -exit - set ::env(CURRENT_ODB) $::env(DESIGN_DIR)/in.odb +set ::env(CURRENT_ODB) $::env(DESIGN_DIR)/in.odb - set save_odb $::env(DESIGN_DIR)/out.odb +set save_odb $::env(DESIGN_DIR)/out.odb - # Remove pins first: nets cannot be removed if they are associated with a pin - remove_components -input $::env(CURRENT_ODB) -output $save_odb - remove_pins -input $save_odb - remove_nets -rx {^in$} -input $save_odb +# Remove pins first: nets cannot be removed if they are associated with a pin +remove_components -input $::env(CURRENT_ODB) -output $save_odb +remove_pins -input $save_odb +remove_nets -rx {^in$} -input $save_odb - set ::env(CURRENT_ODB) $save_odb +set ::env(CURRENT_ODB) $save_odb - try_catch $::env(OPENROAD_BIN) -exit -no_init -python $::env(DESIGN_DIR)/hooks/post_run.py +run_odbpy_script $::env(DESIGN_DIR)/hooks/post_run.py - puts_info "Done." \ No newline at end of file +puts_info "Done."