Setup build images for Pytorch & Tensorflow with base ROCm image

opendatahub-io · Jun 17, 2024 · f64fef2 · f64fef2
1 parent c7040f0
commit f64fef2
Show file tree

Hide file tree

Showing 12 changed files with 592 additions and 0 deletions.
diff --git a/jupyter/amd/pytorch/ubi9-python-3.9/Dockerfile b/jupyter/amd/pytorch/ubi9-python-3.9/Dockerfile
@@ -0,0 +1,27 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+LABEL name="odh-notebook-jupyter--amd-pytorch-ubi9-python-3.9" \
+    summary="Jupyter AMD pytorch notebook image for ODH notebooks" \
+    description="Jupyter AMD pytorch notebook image with base Python 3.9 builder image based on UBI9 for ODH notebooks" \
+    io.k8s.display-name="Jupyter AMD pytorch notebook image for ODH notebooks" \
+    io.k8s.description="Jupyter AMD pytorch notebook image with base Python 3.9 builder image based on UBI9 for ODH notebooks" \
+    authoritative-source-url="https://github.com/opendatahub-io/notebooks" \
+    io.openshift.build.commit.ref="main" \
+    io.openshift.build.source-location="https://github.com/opendatahub-io/notebooks/tree/main/jupyter/amd/pytorch/ubi9-python-3.9" \
+    io.openshift.build.image="quay.io/opendatahub/workbench-images:jupyter-amd-pytorch-ubi9-python-3.9"
+
+# Install Python packages and Jupyterlab extensions from Pipfile.lock
+COPY Pipfile.lock ./
+
+RUN echo "Installing softwares and packages" && \
+    micropipenv install && \
+    rm -f ./Pipfile.lock && \
+    # Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y \
+    sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json && \
+    # Disable announcement plugin of jupyterlab \
+    jupyter labextension disable "@jupyterlab/apputils-extension:announcements"
+
+# Fix permissions to support pip in Openshift environments
+RUN chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \
+    fix-permissions /opt/app-root -P
diff --git a/jupyter/amd/pytorch/ubi9-python-3.9/Pipfile b/jupyter/amd/pytorch/ubi9-python-3.9/Pipfile
@@ -0,0 +1,53 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[[source]]
+url = "https://download.pytorch.org/whl/nightly/rocm6.1/"
+verify_ssl = true
+name = "pytorch"
+
+[dev-packages]
+
+[packages]
+# PyTorch packages
+tensorboard = "~=2.16.2"
+torch = {version = "~=2.4.0", index = "pytorch"}
+torchvision = {version = "~=0.19.0", index = "pytorch"}
+# Datascience and useful extensions
+boto3 = "~=1.34.50"
+kafka-python = "~=2.0.2"
+matplotlib = "~=3.8.3"
+numpy = "~=1.26.4"
+pandas = "~=2.2.0"
+plotly = "~=5.20.0"
+scikit-learn = "~=1.4.0"
+scipy = "~=1.12.0"
+skl2onnx = "~=1.16.0"
+onnxconverter-common = "~=1.13.0" # Required for skl2onnx, as upgraded version is not compatible with protobuf
+codeflare-sdk = "~=0.15.1"
+# DB connectors
+pymongo = "~=4.6.2"
+psycopg = "~=3.1.18"
+pyodbc = "~=5.1.0"
+mysql-connector-python = "~=8.3.0"
+# JupyterLab packages
+odh-elyra = "~=3.16.5"
+jupyterlab = "~=3.6.7" # Wait on upgrade till plugins are ready
+jupyter-bokeh = "~=3.0.7" # Upgrade would bring in jupyterlab 4
+jupyter-server = "~=2.13.0"
+jupyter-server-proxy = "~=4.0.0" # Upgrade would bring in jupyterlab 4
+jupyter-server-terminals = "~=0.5.3"
+jupyterlab-git = "~=0.44.0"
+jupyterlab-lsp = "~=4.2.0"
+jupyterlab-widgets = "~=3.0.10"
+jupyter-resource-usage = "~=0.7.2"
+nbdime = "~=3.2.1"
+nbgitpuller = "~=1.2.0"
+# Base packages
+wheel = "~=0.43.0"
+setuptools = "~=69.2.0"
+
+[requires]
+python_version = "3.9"
diff --git a/jupyter/amd/pytorch/ubi9-python-3.9/kustomize/base/kustomization.yaml b/jupyter/amd/pytorch/ubi9-python-3.9/kustomize/base/kustomization.yaml
@@ -0,0 +1,13 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namePrefix: jupyter-amd-pytorch-ubi9-python-3-9-
+commonLabels:
+  app: jupyter-amd-pytorch-ubi9-python-3-9
+resources:
+  - service.yaml
+  - statefulset.yaml
+images:
+  - name: quay.io/opendatahub/workbench-images
+    newName: quay.io/opendatahub/workbench-images
+    newTag: jupyter-amd-pytorch-ubi9-python-3.9
diff --git a/jupyter/amd/pytorch/ubi9-python-3.9/kustomize/base/service.yaml b/jupyter/amd/pytorch/ubi9-python-3.9/kustomize/base/service.yaml
@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: notebook
+  labels:
+    app: notebook
+spec:
+  type: ClusterIP
+  ports:
+    - port: 8888
+      protocol: TCP
+      targetPort: notebook-port
+  selector:
+    app: notebook
diff --git a/jupyter/amd/pytorch/ubi9-python-3.9/kustomize/base/statefulset.yaml b/jupyter/amd/pytorch/ubi9-python-3.9/kustomize/base/statefulset.yaml
@@ -0,0 +1,58 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: notebook
+  annotations:
+    opendatahub.io/username: jovyan
+  labels:
+    opendatahub.io/user: jovyan
+spec:
+  replicas: 1
+  selector: {}
+  serviceName: notebook
+  template:
+    metadata:
+      labels: {}
+    spec:
+      containers:
+        - name: notebook
+          image: quay.io/opendatahub/workbench-images:jupyter-amd-pytorch-ubi9-python-3.9
+          imagePullPolicy: Always
+          workingDir: /opt/app-root/src
+          env:
+            - name: NOTEBOOK_ARGS
+              value: |-
+                --ServerApp.port=8888
+                --ServerApp.token=''
+                --ServerApp.password=''
+                --ServerApp.base_url=/notebook/opendatahub/jovyan
+                --ServerApp.quit_button=False
+                --ServerApp.tornado_settings={"user":"jovyan","hub_host":"https://opendatahub.io","hub_prefix":"/notebookController/jovyan"}
+          ports:
+            - name: notebook-port
+              protocol: TCP
+              containerPort: 8888
+          livenessProbe:
+            tcpSocket:
+              port: notebook-port
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /notebook/opendatahub/jovyan/api
+              port: notebook-port
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
+          resources:
+            limits:
+              cpu: 500m
+              memory: 2Gi
+            requests:
+              cpu: 500m
+              memory: 2Gi
diff --git a/jupyter/amd/pytorch/ubi9-python-3.9/test/test_notebook.ipynb b/jupyter/amd/pytorch/ubi9-python-3.9/test/test_notebook.ipynb
@@ -0,0 +1,163 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import unittest\n",
+    "import torch\n",
+    "import math\n",
+    "import os\n",
+    "import torchvision.transforms as transforms\n",
+    "from PIL import Image\n",
+    "from torch.utils.tensorboard import SummaryWriter\n",
+    "from platform import python_version\n",
+    "import torchvision.models as models\n",
+    "\n",
+    "class TestPytorchNotebook(unittest.TestCase):\n",
+    "    \n",
+    "    def test_python_version(self):\n",
+    "        expected_major_minor = '3.9'  # Set the expected version (x.y)\n",
+    "        actual_major_minor = '.'.join(python_version().split('.')[:2]) \n",
+    "        self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
+    "    \n",
+    "    def test_torch_version(self):\n",
+    "        expected_major_minor = '2.2'  # Set the expected version (x.y)\n",
+    "        actual_major_minor = '.'.join(torch.__version__.split('.')[:2]) \n",
+    "        self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
+    "        \n",
+    "    def test_tensor_creation(self):\n",
+    "        # Test tensor creation\n",
+    "        tensor = torch.tensor([1, 2, 3])\n",
+    "        self.assertIsInstance(tensor, torch.Tensor)\n",
+    "        \n",
+    "    def test_tensor_operations(self):\n",
+    "        # Test tensor operations\n",
+    "        a = torch.tensor([1, 2, 3])\n",
+    "        b = torch.tensor([4, 5, 6])\n",
+    "        c = a + b\n",
+    "        self.assertTrue(torch.all(torch.eq(c, torch.tensor([5, 7, 9]))))\n",
+    "        \n",
+    "    def test_tensor_shape(self):\n",
+    "        # Test tensor shape\n",
+    "        tensor = torch.tensor([[1, 2, 3], [4, 5, 6]])\n",
+    "        self.assertEqual(tensor.shape, (2, 3))\n",
+    "        \n",
+    "    def test_polynomial_fitting(self):\n",
+    "        dtype = torch.float\n",
+    "        device = torch.device(\"cpu\")\n",
+    "        # device = torch.device(\"cuda:0\") # Uncomment this to run on GPU\n",
+    "\n",
+    "        # Create random input and output data\n",
+    "        x = torch.linspace(-math.pi, math.pi, 2000, device=device, dtype=dtype)\n",
+    "        y = torch.sin(x)\n",
+    "\n",
+    "        # Randomly initialize weights\n",
+    "        a = torch.randn((), device=device, dtype=dtype)\n",
+    "        b = torch.randn((), device=device, dtype=dtype)\n",
+    "        c = torch.randn((), device=device, dtype=dtype)\n",
+    "        d = torch.randn((), device=device, dtype=dtype)\n",
+    "\n",
+    "        learning_rate = 1e-6\n",
+    "        for t in range(2000):\n",
+    "            # Forward pass: compute predicted y\n",
+    "            y_pred = a + b * x + c * x ** 2 + d * x ** 3\n",
+    "\n",
+    "            # Compute and print loss\n",
+    "            loss = (y_pred - y).pow(2).sum().item()\n",
+    "            if t % 100 == 99:\n",
+    "                print(t, loss)\n",
+    "\n",
+    "            # Backprop to compute gradients of a, b, c, d with respect to loss\n",
+    "            grad_y_pred = 2.0 * (y_pred - y)\n",
+    "            grad_a = grad_y_pred.sum()\n",
+    "            grad_b = (grad_y_pred * x).sum()\n",
+    "            grad_c = (grad_y_pred * x ** 2).sum()\n",
+    "            grad_d = (grad_y_pred * x ** 3).sum()\n",
+    "\n",
+    "            # Update weights using gradient descent\n",
+    "            a -= learning_rate * grad_a\n",
+    "            b -= learning_rate * grad_b\n",
+    "            c -= learning_rate * grad_c\n",
+    "            d -= learning_rate * grad_d\n",
+    "\n",
+    "\n",
+    "        print(f'Result: y = {a.item()} + {b.item()} x + {c.item()} x^2 + {d.item()} x^3')\n",
+    "        \n",
+    "    def test_image_transform(self):\n",
+    "            # Define a sample image\n",
+    "            sample_image = Image.new('RGB', (100, 100), color='red')\n",
+    "\n",
+    "            # Define a transformation pipeline\n",
+    "            transform = transforms.Compose([\n",
+    "                transforms.Resize((64, 64)),\n",
+    "                transforms.ToTensor()\n",
+    "            ])\n",
+    "\n",
+    "            # Apply the transformation\n",
+    "            transformed_image = transform(sample_image)\n",
+    "\n",
+    "            # Check if the transformed image is a PyTorch tensor\n",
+    "            self.assertTrue(isinstance(transformed_image, torch.Tensor))\n",
+    "\n",
+    "            # Check if the transformed image has the correct shape\n",
+    "            self.assertEqual(transformed_image.shape, (3, 64, 64))  # Assuming RGB image\n",
+    "\n",
+    "    def test_tensorboard_integration(self):\n",
+    "            # Define a sample image\n",
+    "            sample_image = Image.new('RGB', (100, 100), color='red')\n",
+    "\n",
+    "            # Define a transformation pipeline\n",
+    "            transform = transforms.Compose([\n",
+    "                transforms.Resize((64, 64)),\n",
+    "                transforms.ToTensor()\n",
+    "            ])\n",
+    "\n",
+    "            # Apply the transformation\n",
+    "            transformed_image = transform(sample_image)\n",
+    "\n",
+    "            # Initialize TensorBoard writer\n",
+    "            writer = SummaryWriter(log_dir=\"./logs\")\n",
+    "\n",
+    "            # Add the transformed image to TensorBoard\n",
+    "            writer.add_image(\"Transformed Image\", transformed_image, global_step=0)\n",
+    "\n",
+    "            # Close the writer\n",
+    "            writer.close()\n",
+    "\n",
+    "            # You can manually check the generated TensorBoard logs to verify the image\n",
+    "\n",
+    "            # Assert that the writer has been successfully created and used\n",
+    "            self.assertTrue(isinstance(writer, SummaryWriter))\n",
+    "            \n",
+    "    def test_convert_to_onnx(self):\n",
+    "        # Load a pre-trained PyTorch model (for example, ResNet-18)\n",
+    "        model = models.resnet18(pretrained=True)\n",
+    "        model.eval()\n",
+    "\n",
+    "        # Create a sample input tensor with the appropriate shape\n",
+    "        input_tensor = torch.randn(1, 3, 224, 224)  # (batch_size, channels, height, width)\n",
+    "\n",
+    "        # Export the model to ONNX format\n",
+    "        onnx_path = \"resnet18.onnx\"\n",
+    "        torch.onnx.export(model, input_tensor, onnx_path, verbose=True)\n",
+    "\n",
+    "        # Check if the ONNX file exists\n",
+    "        self.assertTrue(os.path.exists(onnx_path), f\"ONNX file {onnx_path} not found\")\n",
+    "        \n",
+    "suite = unittest.TestLoader().loadTestsFromTestCase(TestPytorchNotebook)\n",
+    "unittest.TextTestRunner().run(suite)"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/jupyter/amd/tensorflow/ubi9-python-3.9/Dockerfile b/jupyter/amd/tensorflow/ubi9-python-3.9/Dockerfile
@@ -0,0 +1,27 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+LABEL name="odh-notebook-cuda-jupyter-tensorflow-ubi9-python-3.9" \
+    summary="Jupyter AMD tensorflow notebook image for ODH notebooks" \
+    description="Jupyter AMD tensorflow notebook image with base Python 3.9 builder image based on UBI9 for ODH notebooks" \
+    io.k8s.display-name="Jupyter AMD tensorflow notebook image for ODH notebooks" \
+    io.k8s.description="Jupyter AMD tensorflow notebook image with base Python 3.9 builder image based on UBI9 for ODH notebooks" \
+    authoritative-source-url="https://github.com/opendatahub-io/notebooks" \
+    io.openshift.build.commit.ref="main" \
+    io.openshift.build.source-location="https://github.com/opendatahub-io/notebooks/tree/main/jupyter/amd/tensorflow/ubi9-python-3.9" \
+    io.openshift.build.image="quay.io/opendatahub/workbench-images:jupyter-amd-tensorflow-ubi9-python-3.9"
+
+# Install Python packages and Jupyterlab extensions from Pipfile.lock
+COPY Pipfile.lock ./
+
+RUN echo "Installing softwares and packages" && micropipenv install && rm -f ./Pipfile.lock
+
+# Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y
+RUN sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json
+
+# Disable announcement plugin of jupyterlab
+RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements"
+
+# Fix permissions to support pip in Openshift environments
+RUN chmod -R g+w /opt/app-root/lib/python3.9/site-packages && \
+    fix-permissions /opt/app-root -P