From 81558ffa108f7518322a8d19abd294fc74536b26 Mon Sep 17 00:00:00 2001 From: Ke Zhu Date: Tue, 6 Oct 2020 20:16:36 -0400 Subject: [PATCH 1/4] Copy kfctl_openshift_tekton_kfserving.v1.1.0.yaml --- ...enshift_tekton_kfserving_appid.v1.1.0.yaml | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 OpenShift/manifests/kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml diff --git a/OpenShift/manifests/kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml b/OpenShift/manifests/kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml new file mode 100644 index 0000000..540b0e8 --- /dev/null +++ b/OpenShift/manifests/kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml @@ -0,0 +1,134 @@ +apiVersion: kfdef.apps.kubeflow.org/v1 +kind: KfDef +metadata: + namespace: kubeflow +spec: + applications: + # openshift specific + - kustomizeConfig: + repoRef: + name: manifests + path: openshift/openshift-scc/base + name: openshift-scc + # istio related components + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/application/istio-stack + name: istio-stack + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/application/cluster-local-gateway + name: cluster-local-gateway + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/application/istio + name: istio + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/application/add-anonymous-user-filter + name: add-anonymous-user-filter + # application + - kustomizeConfig: + repoRef: + name: manifests + path: application/v3 + name: application + # cert-manager + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/application/cert-manager-crds + name: cert-manager-crds + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/application/cert-manager-kube-system-resources + name: cert-manager-kube-system-resources + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/application/cert-manager + name: cert-manager + # bootstrap + # - kustomizeConfig: + # repoRef: + # name: manifests + # path: stacks/openshift/application/bootstrap + # name: bootstrap + # kubeflow apps + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/base + name: kubeflow-apps + # - kustomizeConfig: + # repoRef: + # name: manifests + # path: stacks/openshift/components/admission-webhook + # name: kubeflow-apps + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/components/profile-control-plane + name: kubeflow-apps + # install Tekton Pipelines, if you choose to use OpenShift Pipelines + # pre-installed on your OCP cluster, comment out this component + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/components/tektoncd + name: kubeflow-apps + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/components/kfp-tekton + name: kubeflow-apps + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/components/metadata + name: kubeflow-apps + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/components/notebooks + name: kubeflow-apps + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/components/pytorch-job + name: kubeflow-apps + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/components/tf-job + name: kubeflow-apps + # others: + - kustomizeConfig: + repoRef: + name: manifests + path: metacontroller/base + name: metacontroller + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/application/spark-operator + name: spark-operator + - kustomizeConfig: + repoRef: + name: manifests + path: knative/installs/generic + name: knative + - kustomizeConfig: + repoRef: + name: manifests + path: kfserving/installs/generic + name: kfserving + repos: + - name: manifests + uri: https://github.com/adrian555/manifests/archive/update-manifests-repo-link.tar.gz + version: master From 8c98aadc348958285a45844624926b9e58bbecc6 Mon Sep 17 00:00:00 2001 From: Ke Zhu Date: Tue, 6 Oct 2020 20:18:37 -0400 Subject: [PATCH 2/4] Add multi-tenancy feature via AppID --- .../kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OpenShift/manifests/kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml b/OpenShift/manifests/kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml index 540b0e8..e50d46a 100644 --- a/OpenShift/manifests/kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml +++ b/OpenShift/manifests/kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml @@ -29,8 +29,8 @@ spec: - kustomizeConfig: repoRef: name: manifests - path: stacks/openshift/application/add-anonymous-user-filter - name: add-anonymous-user-filter + path: stacks/ibm/application/oidc-authservice-for-appid + name: oidc-authservice # application - kustomizeConfig: repoRef: From 51f62891b4fd0a20aac947d70ffa6a0e6a2b9378 Mon Sep 17 00:00:00 2001 From: Ke Zhu Date: Tue, 6 Oct 2020 20:19:51 -0400 Subject: [PATCH 3/4] Experiment from a branch of a fork --- .../kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OpenShift/manifests/kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml b/OpenShift/manifests/kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml index e50d46a..c7b2e29 100644 --- a/OpenShift/manifests/kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml +++ b/OpenShift/manifests/kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml @@ -29,7 +29,7 @@ spec: - kustomizeConfig: repoRef: name: manifests - path: stacks/ibm/application/oidc-authservice-for-appid + path: stacks/openshift/application/oidc-authservice-for-appid name: oidc-authservice # application - kustomizeConfig: @@ -130,5 +130,5 @@ spec: name: kfserving repos: - name: manifests - uri: https://github.com/adrian555/manifests/archive/update-manifests-repo-link.tar.gz + uri: https://github.com/IBM/manifests/archive/master.tar.gz version: master From 97d3112a6b4f5fab5d0f9bf92884fe970932c5eb Mon Sep 17 00:00:00 2001 From: Ke Zhu Date: Tue, 27 Oct 2020 22:46:01 -0400 Subject: [PATCH 4/4] Document deployment steps of multi-tenant Kubeflow --- OpenShift/README.md | 4 ++ OpenShift/manifests/README-appid.md | 67 +++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 OpenShift/manifests/README-appid.md diff --git a/OpenShift/README.md b/OpenShift/README.md index 5db54f0..55fb4d1 100644 --- a/OpenShift/README.md +++ b/OpenShift/README.md @@ -29,3 +29,7 @@ This guide describes how to deploy Kubeflow on OpenShift clusters. There are two ## Deploy Kubeflow with Knative and KFServing For users who want to run KFserving service along with Kubeflow on OpenShift clusters, follow the [KfServing on OpenShift](manifests/README-kfserving.md) guide to deploy. + +## Multi-tenant Kubeflow on OpenShift with IBM Cloud AppID + + For users who want to run KFserving service along with Multi-tenant Kubeflow on OpenShift clusters, follow the [Multi-tenant Kubeflow on OpenShift with IBM Cloud AppID](manifests/README-appid.md) guide to deploy. \ No newline at end of file diff --git a/OpenShift/manifests/README-appid.md b/OpenShift/manifests/README-appid.md new file mode 100644 index 0000000..8a4aede --- /dev/null +++ b/OpenShift/manifests/README-appid.md @@ -0,0 +1,67 @@ +## Multi-tenant Kubeflow on OpenShift with IBM Cloud AppID + +This guide is based on [KfServing on OpenShift](./README-kfserving.md) with multi-tenancy feature enabled by IBM Cloud AppID. + +### Prerequisites + +1. Follow the [Prepare OpenShift cluster environment](./README.md#prepare-openshift-cluster-environment) to set up the cluster environment. +2. FQDN of OpenShift Route of istio ingress gateway. +3. Provisioning an AppID instance from IBM Cloud. It can start with the Lite plan, but will need the Graduated tier once you need more than 1000 authentication events per month. +4. Create an application with type reguarwebapp under the provioned AppID instance. Make sure the caope contains email and retrieve the following configuration parameters from your AppID. They will be used to configure the OIDC auth service: + * clientId + * secret + * oAuthServerUrl + +### Configuration + +1. Create the namespace `istio-system` if not exist: +```SHELL +kubectl create namespace istio-system +``` +2. Create a secret prior to kubeflow deployment by filling parameters accordingly: +```SHELL +kubectl create secret generic appid-application-configuration -n istio-system \ + --from-literal=clientId= \ + --from-literal=secret= \ + --from-literal=oAuthServerUrl= \ + --from-literal=oidcRedirectUrl=https://istio-ingressgateway-istio-system./login/oidc +``` +* `` - fill in the value of `oAuthServerUrl` +* `` - fill in the value of `clientId` +* `` - fill in the value of `secret` +* `` - fill in the value of _Ingress Subdomain_ out of cluster +details by running command `ibmcloud ks cluster get -c ` where replace `` with your OpenShift cluster name. + +### Deploy Kubeflow with KfServing + +Choose [kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml](./kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml) to deploy the required components for multi-tenant Kubeflow with Tekton backend. + +```shell +export KFDEF_DIR= +mkdir -p ${KFDEF_DIR} +cd ${KFDEF_DIR} +wget https://raw.githubusercontent.com/IBM/KubeflowDojo/master/OpenShift/manifests/kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml +``` + +If you choose to leverage the pre-installed OpenShift Pipelines as the Tekton backend, please comment out these lines from the above configuration file. + +```yaml + - kustomizeConfig: + repoRef: + name: manifests + path: stacks/openshift/components/tektoncd + name: kubeflow-apps +``` + +Run following command to deploy Kubeflow: + +```shell +kfctl apply -V -f kfctl_openshift_tekton_kfserving_appid.v1.1.0.yaml +``` + +### Secure istio ingress gateway with HTTPS + +Notice that it uses HTTPS for the value of `oidcRedirectUrl` during configuration, which +requires additional steps after deploying Kubeflow: +1. enable [TLS passthrough](https://docs.openshift.com/enterprise/3.0/architecture/core_concepts/routes.html#passthrough-termination) mode for the route. +2. expose kubeflow dashboard over HTTPS by following steps of [this section](https://www.kubeflow.org/docs/ibm/deploy/authentication/#exposing-the-kubeflow-dashboard-with-dns-and-tls-termination). \ No newline at end of file