{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Run an EasyVVUQ campaign to analyze the sensitivity for the Ishigami function using MCsampler\n", "\n", "This is done with the `MCsampler` that uses the Saltelli algorithm [1] to compute the Sobol indices. If `n_mc_samples` is the number of Monte Carlo samples specified in the `MCSampler`, the Saltelli algorithm uses `n_mc_samples * (d + 2)` samples to compute the Sobol indices, where `d` is the number of random input variables. Bootstrapping is used to compute confidence intervals on the Sobol index estimates. \n", "\n", "[1] A. Saltelli, *Making best use of model evaluations to compute sensitivity indices*, Computer Physics Communications, 2002." ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2021-06-07T14:15:21.463066Z", "start_time": "2021-06-07T14:15:16.010020Z" }, "code_folding": [ 0 ], "execution": { "iopub.execute_input": "2025-07-18T11:23:05.392873Z", "iopub.status.busy": "2025-07-18T11:23:05.392470Z", "iopub.status.idle": "2025-07-18T11:23:23.747814Z", "shell.execute_reply": "2025-07-18T11:23:23.746644Z", "shell.execute_reply.started": "2025-07-18T11:23:05.392852Z" } }, "outputs": [], "source": [ "# Run an EasyVVUQ campaign to analyze tgohe sensitivity for the Ishigami function\n", "# This is done with SC.\n", "%matplotlib inline\n", "import os\n", "import easyvvuq as uq\n", "import chaospy as cp\n", "import pickle\n", "import numpy as np\n", "import matplotlib.pylab as plt\n", "import time\n", "import pandas as pd\n", "from collections import OrderedDict" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2021-06-07T14:15:21.476480Z", "start_time": "2021-06-07T14:15:21.468639Z" }, "execution": { "iopub.execute_input": "2025-07-18T11:23:23.760203Z", "iopub.status.busy": "2025-07-18T11:23:23.758332Z", "iopub.status.idle": "2025-07-18T11:23:23.769010Z", "shell.execute_reply": "2025-07-18T11:23:23.768481Z", "shell.execute_reply.started": "2025-07-18T11:23:23.759380Z" } }, "outputs": [ { "data": { "text/plain": [ "'1.26.4'" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "np.__version__" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2021-06-07T14:15:21.500137Z", "start_time": "2021-06-07T14:15:21.486267Z" }, "code_folding": [ 0, 1 ], "execution": { "iopub.execute_input": "2025-07-18T11:23:23.769909Z", "iopub.status.busy": "2025-07-18T11:23:23.769747Z", "iopub.status.idle": "2025-07-18T11:23:23.774845Z", "shell.execute_reply": "2025-07-18T11:23:23.774465Z", "shell.execute_reply.started": "2025-07-18T11:23:23.769893Z" } }, "outputs": [], "source": [ "# Define the Ishigami function\n", "def ishigamiSA(a,b):\n", " '''Exact sensitivity indices of the Ishigami function for given a and b.\n", " From https://openturns.github.io/openturns/master/examples/meta_modeling/chaos_ishigami.html\n", " '''\n", " var = 1.0/2 + a**2/8 + b*np.pi**4/5 + b**2*np.pi**8/18\n", " S1 = (1.0/2 + b*np.pi**4/5+b**2*np.pi**8/50)/var\n", " S2 = (a**2/8)/var\n", " S3 = 0\n", " S13 = b**2*np.pi**8/2*(1.0/9-1.0/25)/var\n", " exact = {\n", " 'expectation' : a/2,\n", " 'variance' : var,\n", " 'S1' : (1.0/2 + b*np.pi**4/5+b**2*np.pi**8.0/50)/var,\n", " 'S2' : (a**2/8)/var,\n", " 'S3' : 0,\n", " 'S12' : 0,\n", " 'S23' : 0,\n", " 'S13' : S13,\n", " 'S123' : 0,\n", " 'ST1' : S1 + S13,\n", " 'ST2' : S2,\n", " 'ST3' : S3 + S13\n", " }\n", " return exact\n", "\n", "Ishigami_a = 7.0\n", "Ishigami_b = 0.1\n", "exact = ishigamiSA(Ishigami_a, Ishigami_b)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2021-06-07T14:15:21.510035Z", "start_time": "2021-06-07T14:15:21.505751Z" }, "code_folding": [ 0 ], "execution": { "iopub.execute_input": "2025-07-18T11:23:23.775798Z", "iopub.status.busy": "2025-07-18T11:23:23.775605Z", "iopub.status.idle": "2025-07-18T11:23:23.778410Z", "shell.execute_reply": "2025-07-18T11:23:23.778025Z", "shell.execute_reply.started": "2025-07-18T11:23:23.775782Z" } }, "outputs": [], "source": [ "# define a model to run the Ishigami code directly from python, expecting a dictionary and returning a dictionary\n", "def run_ishigami_model(input):\n", " import Ishigami\n", " qois = [\"Ishigami\"]\n", " del input['out_file']\n", " return {qois[0]: Ishigami.evaluate(**input)}" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2021-06-07T14:15:21.520594Z", "start_time": "2021-06-07T14:15:21.515444Z" }, "code_folding": [ 0 ], "execution": { "iopub.execute_input": "2025-07-18T11:23:23.779081Z", "iopub.status.busy": "2025-07-18T11:23:23.778918Z", "iopub.status.idle": "2025-07-18T11:23:23.783286Z", "shell.execute_reply": "2025-07-18T11:23:23.782864Z", "shell.execute_reply.started": "2025-07-18T11:23:23.779065Z" } }, "outputs": [], "source": [ "# Define parameter space\n", "def define_params():\n", " return {\n", " \"x1\": {\"type\": \"float\", \"min\": -np.pi, \"max\": np.pi, \"default\": 0.0},\n", " \"x2\": {\"type\": \"float\", \"min\": -np.pi, \"max\": np.pi, \"default\": 0.0},\n", " \"x3\": {\"type\": \"float\", \"min\": -np.pi, \"max\": np.pi, \"default\": 0.0},\n", " \"a\": {\"type\": \"float\", \"min\": Ishigami_a, \"max\": Ishigami_a, \"default\": Ishigami_a},\n", " \"b\": {\"type\": \"float\", \"min\": Ishigami_b, \"max\": Ishigami_b, \"default\": Ishigami_b},\n", " \"out_file\": {\"type\": \"string\", \"default\": \"output.csv\"}\n", " }" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2021-06-07T14:15:21.530671Z", "start_time": "2021-06-07T14:15:21.525407Z" }, "code_folding": [ 0 ], "execution": { "iopub.execute_input": "2025-07-18T11:23:23.784258Z", "iopub.status.busy": "2025-07-18T11:23:23.784027Z", "iopub.status.idle": "2025-07-18T11:23:23.786977Z", "shell.execute_reply": "2025-07-18T11:23:23.786638Z", "shell.execute_reply.started": "2025-07-18T11:23:23.784239Z" } }, "outputs": [], "source": [ "# Define parameter space\n", "def define_vary():\n", " return {\n", " \"x1\": cp.Uniform(-np.pi, np.pi),\n", " \"x2\": cp.Uniform(-np.pi, np.pi),\n", " \"x3\": cp.Uniform(-np.pi, np.pi)\n", " }" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2021-06-07T14:15:21.547702Z", "start_time": "2021-06-07T14:15:21.534015Z" }, "code_folding": [ 0 ], "execution": { "iopub.execute_input": "2025-07-18T11:23:23.787836Z", "iopub.status.busy": "2025-07-18T11:23:23.787574Z", "iopub.status.idle": "2025-07-18T11:23:23.793974Z", "shell.execute_reply": "2025-07-18T11:23:23.793584Z", "shell.execute_reply.started": "2025-07-18T11:23:23.787820Z" } }, "outputs": [], "source": [ "# Set up and run a campaign\n", "def run_campaign(n_mc_samples, use_files=False):\n", "\n", " times = np.zeros(7)\n", "\n", " time_start = time.time()\n", " time_start_whole = time_start\n", "\n", " work_dir = '/tmp'\n", " \n", " # Set up a fresh campaign called \"Ishigami_sc.\"\n", " my_campaign = uq.Campaign(name='Ishigami_mc.', work_dir=work_dir)\n", "\n", " # Create an encoder and decoder for SC test app\n", " if use_files:\n", " encoder = uq.encoders.GenericEncoder(template_fname='Ishigami.template',\n", " delimiter='$',\n", " target_filename='Ishigami_in.json')\n", "\n", " decoder = uq.decoders.SimpleCSV(target_filename=\"output.csv\",\n", " output_columns=[\"Ishigami\"])\n", "\n", " execute = uq.actions.ExecuteLocal('python3 %s/Ishigami.py Ishigami_in.json' % (os.getcwd()))\n", "\n", " actions = uq.actions.Actions(uq.actions.CreateRunDirectory(work_dir), \n", " uq.actions.Encode(encoder), execute, uq.actions.Decode(decoder))\n", " else:\n", " actions = uq.actions.Actions(uq.actions.ExecutePython(run_ishigami_model))\n", "\n", " # Add the app (automatically set as current app)\n", " my_campaign.add_app(name=\"Ishigami\", params=define_params(), actions=actions)\n", "\n", " # Create the sampler\n", " time_end = time.time()\n", " times[1] = time_end-time_start\n", " print('Time for phase 1 = %.3f' % (times[1]))\n", "\n", " time_start = time.time()\n", " # Associate a sampler with the campaign\n", " # Latin HyperCube\n", " sampler = uq.sampling.MCSampler(vary=define_vary(), n_mc_samples=n_mc_samples, rule='sobol')\n", " # QMC method with Sobol sequence\n", " #sampler = uq.sampling.QMCSampler(vary=define_vary(), n_mc_samples=n_mc_samples)\n", " my_campaign.set_sampler(sampler)\n", "\n", " # Will draw all (of the finite set of samples)\n", " my_campaign.draw_samples()\n", " print('Number of samples = %s' % my_campaign.get_active_sampler().count)\n", "\n", " time_end = time.time()\n", " times[2] = time_end-time_start\n", " print('Time for phase 2 = %.3f' % (times[2]))\n", "\n", " time_start = time.time()\n", " # Run the cases\n", " my_campaign.execute(sequential=True).collate(progress_bar=True)\n", "\n", " time_end = time.time()\n", " times[3] = time_end-time_start\n", " print('Time for phase 3 = %.3f' % (times[3]))\n", "\n", " time_start = time.time()\n", " # Get the results\n", " results_df = my_campaign.get_collation_result()\n", "\n", " time_end = time.time()\n", " times[4] = time_end-time_start\n", " print('Time for phase 4 = %.3f' % (times[4]))\n", "\n", " time_start = time.time()\n", " # Post-processing analysis\n", " results = my_campaign.analyse(qoi_cols=[\"Ishigami\"])\n", " \n", " time_end = time.time()\n", " times[5] = time_end-time_start\n", " print('Time for phase 5 = %.3f' % (times[5]))\n", "\n", " time_start = time.time()\n", " # Save the results\n", " #pickle.dump(results, open('Ishigami_results.pickle','bw'))\n", " time_end = time.time()\n", " times[6] = time_end-time_start\n", " print('Time for phase 6 = %.3f' % (times[6]))\n", "#\n", " times[0] = time_end - time_start_whole\n", "\n", " return results_df, results, times, n_mc_samples, my_campaign.get_active_sampler().count" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2021-06-07T14:17:40.849526Z", "start_time": "2021-06-07T14:15:21.555355Z" }, "code_folding": [ 0 ], "execution": { "iopub.execute_input": "2025-07-18T11:23:23.794600Z", "iopub.status.busy": "2025-07-18T11:23:23.794473Z", "iopub.status.idle": "2025-07-18T11:25:11.822897Z", "shell.execute_reply": "2025-07-18T11:25:11.822247Z", "shell.execute_reply.started": "2025-07-18T11:23:23.794590Z" }, "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Time for phase 1 = 0.045\n", "Number of samples = 500\n", "Time for phase 2 = 0.120\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████████| 500/500 [00:00<00:00, 6392.88it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Time for phase 3 = 0.110\n", "Time for phase 4 = 0.009\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Time for phase 5 = 0.152\n", "Time for phase 6 = 0.000\n", "Time for phase 1 = 0.007\n", "Number of samples = 2500\n", "Time for phase 2 = 0.606\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████| 2500/2500 [00:00<00:00, 6669.05it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Time for phase 3 = 0.421\n", "Time for phase 4 = 0.028\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Time for phase 5 = 0.806\n", "Time for phase 6 = 0.000\n", "Time for phase 1 = 0.006\n", "Number of samples = 5000\n", "Time for phase 2 = 1.144\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|█████████████████████████████████████| 5000/5000 [00:00<00:00, 6494.61it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Time for phase 3 = 0.859\n", "Time for phase 4 = 0.115\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Time for phase 5 = 1.430\n", "Time for phase 6 = 0.000\n", "Time for phase 1 = 0.006\n", "Number of samples = 10000\n", "Time for phase 2 = 2.276\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████| 10000/10000 [00:01<00:00, 6183.05it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Time for phase 3 = 1.861\n", "Time for phase 4 = 0.173\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Time for phase 5 = 3.428\n", "Time for phase 6 = 0.000\n", "Time for phase 1 = 0.028\n", "Number of samples = 25000\n", "Time for phase 2 = 6.495\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████| 25000/25000 [00:04<00:00, 5798.93it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Time for phase 3 = 4.972\n", "Time for phase 4 = 0.484\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Time for phase 5 = 8.124\n", "Time for phase 6 = 0.000\n", "Time for phase 1 = 0.018\n", "Number of samples = 40000\n", "Time for phase 2 = 9.597\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████| 40000/40000 [00:07<00:00, 5474.42it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Time for phase 3 = 8.179\n", "Time for phase 4 = 0.718\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Time for phase 5 = 14.460\n", "Time for phase 6 = 0.000\n", "Time for phase 1 = 0.018\n", "Number of samples = 50000\n", "Time for phase 2 = 11.891\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|███████████████████████████████████| 50000/50000 [00:09<00:00, 5116.26it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Time for phase 3 = 11.066\n", "Time for phase 4 = 0.920\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Vectorized bootstrapping\n", "Time for phase 5 = 17.451\n", "Time for phase 6 = 0.000\n" ] } ], "source": [ "# Calculate the results for a range of MC points\n", "\n", "d = len(define_vary())\n", "\n", "R = {}\n", "for n_mc_samples in [100, 500, 1000, 2000, 5000, 8000, 10000]:\n", " # the true number of MC samples used by Saltelli's algorithm\n", " n = n_mc_samples * (d + 2)\n", " R[n] = {}\n", " (R[n]['results_df'], \n", " R[n]['results'], \n", " R[n]['times'], \n", " R[n]['order'], \n", " R[n]['number_of_samples']) = run_campaign(n_mc_samples=n_mc_samples, use_files=False)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2021-06-07T14:17:41.753581Z", "start_time": "2021-06-07T14:17:41.285527Z" }, "code_folding": [ 0 ], "execution": { "iopub.execute_input": "2025-07-18T11:25:11.825700Z", "iopub.status.busy": "2025-07-18T11:25:11.825518Z", "iopub.status.idle": "2025-07-18T11:25:11.850561Z", "shell.execute_reply": "2025-07-18T11:25:11.850326Z", "shell.execute_reply.started": "2025-07-18T11:25:11.825670Z" } }, "outputs": [ { "data": { "text/html": [ "
| \n", " | Total | \n", "Phase 1 | \n", "Phase 2 | \n", "Phase 3 | \n", "Phase 4 | \n", "Phase 5 | \n", "Phase 6 | \n", "
|---|---|---|---|---|---|---|---|
| 100 | \n", "0.435259 | \n", "0.044919 | \n", "0.119578 | \n", "0.109621 | \n", "0.008567 | \n", "0.152388 | \n", "0.000000e+00 | \n", "
| 500 | \n", "1.868456 | \n", "0.007364 | \n", "0.606182 | \n", "0.420718 | \n", "0.027718 | \n", "0.806315 | \n", "0.000000e+00 | \n", "
| 1000 | \n", "3.553871 | \n", "0.006349 | \n", "1.144071 | \n", "0.859180 | \n", "0.114510 | \n", "1.429628 | \n", "0.000000e+00 | \n", "
| 2000 | \n", "7.744114 | \n", "0.005847 | \n", "2.275557 | \n", "1.861108 | \n", "0.173382 | \n", "3.428053 | \n", "9.536743e-07 | \n", "
| 5000 | \n", "20.103047 | \n", "0.027934 | \n", "6.494843 | \n", "4.971630 | \n", "0.484398 | \n", "8.123843 | \n", "0.000000e+00 | \n", "
| 8000 | \n", "32.972410 | \n", "0.017668 | \n", "9.597153 | \n", "8.179419 | \n", "0.717565 | \n", "14.460245 | \n", "0.000000e+00 | \n", "
| 10000 | \n", "41.345705 | \n", "0.017865 | \n", "11.890518 | \n", "11.065725 | \n", "0.919810 | \n", "17.451130 | \n", "0.000000e+00 | \n", "