diff --git a/vm-ubuntu-24.04/install-p4dev-v8.sh b/vm-ubuntu-24.04/install-p4dev-v8.sh index 1d80fd5d..287286a3 100755 --- a/vm-ubuntu-24.04/install-p4dev-v8.sh +++ b/vm-ubuntu-24.04/install-p4dev-v8.sh @@ -140,6 +140,32 @@ max_of_list() { echo ${max} } +dump_python_lib_info() { + local output_dir=$1 + set +e + mkdir -p ${output_dir} + outf=${output_dir}/dirs.txt + echo "All directories named site-packages or dist-packages:" > ${outf} + find / -name site-packages -o -name dist-packages | sort >> ${outf} + echo "" >> ${outf} + echo "ls -la on each such directory:" >> ${outf} + for d in $(find / -name site-packages -o -name dist-packages | sort) + do + echo $d >> ${outf} + ls -la $d >> ${outf} + done + #echo "" >> ${outf} + #echo "ls -laR on each such directory:" >> ${outf} + for d in $(find / -name site-packages -o -name dist-packages | sort) + do + outf="${output_dir}/ls-laR-of-$(echo "$d" | tr '/' '-')" + cp /dev/null ${outf} + #echo $d >> ${outf} + ls -laR $d >> ${outf} + done + set -e +} + # Change this to a lower value if you do not like all this extra debug # output. It is occasionally useful to debug why Python package # install files, or other files installed system-wide, are not going @@ -474,6 +500,10 @@ pip -V || echo "No such command in PATH: pip" pip2 -V || echo "No such command in PATH: pip2" pip3 -V || echo "No such command in PATH: pip3" +DEBUG_DUMP_DIR="${HOME}/install-p4dev-dumpdir" +mkdir -p ${DEBUG_DUMP_DIR} +dump_python_lib_info "${DEBUG_DUMP_DIR}/000-init" + # On new systems if you have never checked repos you should do that first # Install a few packages (vim is not strictly necessary -- installed for @@ -498,6 +528,7 @@ fi TIME_AUTOTOOLS_START=$(date +%s) if [ "${ID}" = "ubuntu" ] then + dump_python_lib_info "${DEBUG_DUMP_DIR}/003-just-before-python3-pip-install" sudo apt-get --yes install \ autoconf automake libtool curl make g++ unzip \ pkg-config python3-pip python3-venv @@ -508,6 +539,8 @@ then pkg-config python3-pip fi +dump_python_lib_info "${DEBUG_DUMP_DIR}/005-after-python-apt-installs" + if [ \( "${ID}" = "ubuntu" -a "${VERSION_ID}" = "20.04" \) -o \( "${ID}" = "fedora" -a "${VERSION_ID}" = "35" \) ] then if [ -d automake-1.16.5 ] @@ -561,11 +594,21 @@ TIME_AUTOTOOLS_END=$(date +%s) echo "autotools : $(($TIME_AUTOTOOLS_END-$TIME_AUTOTOOLS_START)) sec" DISK_USED_AFTER_AUTOTOOLS=`get_used_disk_space_in_mbytes` +# Install uv +curl -LsSf https://astral.sh/uv/install.sh | sh +echo $PATH +source $HOME/.local/bin/env +echo $PATH +which uv +uv pip list + # Create a new Python virtual environment using venv. Later we will # attempt to ensure that all new Python packages installed are # installed into this virtual environment, not into system-wide # directories like /usr/local/bin -python3 -m venv "${PYTHON_VENV}" +dump_python_lib_info "${DEBUG_DUMP_DIR}/008-just-before-venv-creation" +#python3 -m venv "${PYTHON_VENV}" +uv venv "${PYTHON_VENV}" source "${PYTHON_VENV}/bin/activate" pip -V || echo "No such command in PATH: pip" @@ -579,8 +622,11 @@ pip3 -V || echo "No such command in PATH: pip3" pip list || echo "Some error occurred attempting to run command: pip" pip3 list || echo "Some error occurred attempting to run command: pip3" +uv pip list + cd "${INSTALL_DIR}" debug_dump_many_install_files ${INSTALL_DIR}/usr-local-1-before-protobuf.txt +dump_python_lib_info "${DEBUG_DUMP_DIR}/010-before-installing-grpc" set +x echo "------------------------------------------------------------" @@ -597,10 +643,12 @@ then sudo apt-get --yes install libprotobuf-dev protobuf-compiler protobuf-compiler-grpc libgrpc-dev libgrpc++-dev if [ "${PROTOBUF_VERSION_FOR_PIP}" != "" ] then - pip3 install protobuf==${PROTOBUF_VERSION_FOR_PIP} + dump_python_lib_info "${DEBUG_DUMP_DIR}/013-just-before-installing-protobuf-via-pip" + uv pip install protobuf==${PROTOBUF_VERSION_FOR_PIP} + dump_python_lib_info "${DEBUG_DUMP_DIR}/015-after-installing-protobuf-via-pip" fi TIME_GRPC_INSTALL_END=$(date +%s) - pip3 list + uv pip list else # Do not bother installing protobuf package from source code, as # whatever parts of protobuf we need is installed as a result of @@ -608,7 +656,9 @@ else # protobuf package using pip. if [ "${PROTOBUF_VERSION_FOR_PIP}" != "" ] then - pip3 install protobuf==${PROTOBUF_VERSION_FOR_PIP} + dump_python_lib_info "${DEBUG_DUMP_DIR}/013-just-before-installing-protobuf-via-pip" + uv pip install protobuf==${PROTOBUF_VERSION_FOR_PIP} + dump_python_lib_info "${DEBUG_DUMP_DIR}/015-after-installing-protobuf-via-pip" fi cd "${INSTALL_DIR}" @@ -673,9 +723,11 @@ else # https://github.com/grpc/grpc/issues/30524 cmake ../.. -DgRPC_SSL_PROVIDER=package make + dump_python_lib_info "${DEBUG_DUMP_DIR}/013-just-before-grpc-sudo-make-install" sudo make install cd ../.. sudo ldconfig + dump_python_lib_info "${DEBUG_DUMP_DIR}/020-after-installing-grpc-from-source" # Without the following command, later the command 'pkg-config # --cflags grpc' fails, at least on Ubuntu 23.10 after building # grpc v1.54.2 @@ -733,6 +785,8 @@ echo "start install PI:" set -x date +dump_python_lib_info "${DEBUG_DUMP_DIR}/025-before-PI" + TIME_PI_CLONE_START=$(date +%s) TIME_PI_CLONE_END=$(date +%s) TIME_PI_INSTALL_START=$(date +%s) @@ -783,7 +837,9 @@ else /usr/local/bin/protoc --version set -e make + dump_python_lib_info "${DEBUG_DUMP_DIR}/027-PI-just-before-sudo-make-install" sudo make install + dump_python_lib_info "${DEBUG_DUMP_DIR}/029-PI-just-after-sudo-make-install" DISK_USED_BEFORE_PI_CLEANUP=`get_used_disk_space_in_mbytes` if [ ${CLEAN_UP_AS_WE_GO} -eq 1 ] @@ -807,6 +863,8 @@ echo "end install PI:" set -x date +dump_python_lib_info "${DEBUG_DUMP_DIR}/030-before-behavioral-model" + cd "${INSTALL_DIR}" debug_dump_many_install_files ${INSTALL_DIR}/usr-local-4-after-PI.txt @@ -853,7 +911,9 @@ else patch -p1 < "${PATCH_DIR}/behavioral-model-support-venv-2026-apr.patch" # This command installs Thrift, which I want to include in my build of # simple_switch_grpc + dump_python_lib_info "${DEBUG_DUMP_DIR}/031-behavioral-model-just-before-install_deps" ./install_deps.sh + dump_python_lib_info "${DEBUG_DUMP_DIR}/035-behavioral-model-just-after-install_deps" # simple_switch_grpc README.md says to configure and build the bmv2 # code first, using these commands: ./autogen.sh @@ -866,11 +926,14 @@ else PKG_CONFIG_PATH=/usr/local/lib/pkgconfig ./configure --with-pi --with-thrift ${configure_python_prefix} 'CXXFLAGS=-O0 -g' fi make + dump_python_lib_info "${DEBUG_DUMP_DIR}/038-behavioral-model-just-before-sudo-make-install-strip" sudo make install-strip sudo ldconfig + dump_python_lib_info "${DEBUG_DUMP_DIR}/040-after-behavioral-model-install" # 'sudo make install-strip' installs several files in ${PYTHON_VENV} # with root owner. Change them to be owned by the regular user id. change_owner_and_group_of_venv_lib_python3_files ${PYTHON_VENV} + dump_python_lib_info "${DEBUG_DUMP_DIR}/045-after-behavioral-model-change_owner" DISK_USED_BEFORE_BMV2_CLEANUP=`get_used_disk_space_in_mbytes` if [ ${CLEAN_UP_AS_WE_GO} -eq 1 ] then @@ -894,6 +957,8 @@ date cd "${INSTALL_DIR}" debug_dump_many_install_files ${INSTALL_DIR}/usr-local-5-after-behavioral-model.txt +dump_python_lib_info "${DEBUG_DUMP_DIR}/050-before-p4c" + set +x echo "------------------------------------------------------------" echo "Installing p4lang/p4c" @@ -928,8 +993,10 @@ fi # TODO: It appears that some changes were made from scapy 2.5.0 to # 2.6.0 that require changes in P4 open source tools in order to use # version 2.6.0. Until those changes are made, install scapy 2.5.0. -pip3 install scapy==2.5.0 ply -pip3 list +uv pip install scapy==2.5.0 ply +uv pip list + +dump_python_lib_info "${DEBUG_DUMP_DIR}/055-before-p4c-after-install-scapy" DISK_USED_BEFORE_P4C_CLEANUP=`get_used_disk_space_in_mbytes` if [ -d p4c ] @@ -956,7 +1023,9 @@ else cmake .. -DCMAKE_BUILD_TYPE=Release ${P4C_CMAKE_OPTS} MAX_PARALLEL_JOBS=`max_parallel_jobs 2048` make -j${MAX_PARALLEL_JOBS} + dump_python_lib_info "${DEBUG_DUMP_DIR}/058-p4c-just-before-sudo-make-install-strip" sudo make install/strip + dump_python_lib_info "${DEBUG_DUMP_DIR}/060-p4c-just-after-sudo-make-install-strip" sudo ldconfig DISK_USED_BEFORE_P4C_CLEANUP=`get_used_disk_space_in_mbytes` if [ ${CLEAN_UP_AS_WE_GO} -eq 1 -a ${KEEP_P4C_BUILD_FOR_TESTING} -eq 0 ] @@ -1014,7 +1083,9 @@ then sudo mv /etc/sudoers.d/sudoers-dotfiles /etc/sudoers.d/sudoers-dotfiles.orig RESTORE_SUDOERS_FILE=1 fi +dump_python_lib_info "${DEBUG_DUMP_DIR}/065-mininet-just-before-install" PYTHON=python3 ./mininet/util/install.sh -nw +dump_python_lib_info "${DEBUG_DUMP_DIR}/070-mininet-just-after-install" if [ ${RESTORE_SUDOERS_FILE} -eq 1 ] then sudo mv /etc/sudoers.d/sudoers-dotfiles.orig /etc/sudoers.d/sudoers-dotfiles @@ -1046,13 +1117,16 @@ TIME_PTF_START=$(date +%s) # Ubuntu 22.04. #sudo pip3 install pypcap +dump_python_lib_info "${DEBUG_DUMP_DIR}/075-before-ptf" git clone https://github.com/p4lang/ptf cd ptf if [ "x${INSTALL_PTF_SOURCE_VERSION}" != "x" ]; then git checkout ${INSTALL_PTF_SOURCE_VERSION} fi git log -n 1 -pip install . +dump_python_lib_info "${DEBUG_DUMP_DIR}/078-ptf-just-before-pip-install" +uv pip install . +dump_python_lib_info "${DEBUG_DUMP_DIR}/080-ptf-just-after-pip-install" TIME_PTF_END=$(date +%s) echo "p4lang/ptf : $(($TIME_PTF_END-$TIME_PTF_START)) sec" @@ -1079,7 +1153,9 @@ elif [ "${ID}" = "fedora" ] then sudo dnf -y install gflags-devel net-tools fi -pip3 install psutil crcmod +dump_python_lib_info "${DEBUG_DUMP_DIR}/084-before-pip-install-psutil-crcmod" +uv pip install psutil crcmod +dump_python_lib_info "${DEBUG_DUMP_DIR}/085-after-pip-install-psutil-crcmod" # Install p4runtime-shell from source repo, with a slightly modified # setup.cfg file so that it allows us to keep the version of the @@ -1092,15 +1168,17 @@ pip3 install psutil crcmod # First install a known working version of the grpcio package, because # otherwise installing p4runtime-shell packages will likely pick some # very recent version of grpcio that may cause trouble. -pip3 install wheel +uv pip install wheel +dump_python_lib_info "${DEBUG_DUMP_DIR}/090-after-pip-install-wheel" if [ "${ID}" == "ubuntu" -a "${VERSION_ID}" == "24.04" ] then # Version 1.51.3 fails to install on Ubuntu 24.04 as of # 2024-May-20. - pip3 install grpcio==1.59.3 + uv pip install grpcio==1.59.3 else - pip3 install grpcio==1.51.3 + uv pip install grpcio==1.51.3 fi +dump_python_lib_info "${DEBUG_DUMP_DIR}/095-after-pip-install-grpcio" git clone https://github.com/p4lang/p4runtime-shell cd p4runtime-shell @@ -1110,7 +1188,9 @@ fi git log -n 1 PATCH_DIR="${THIS_SCRIPT_DIR_ABSOLUTE}/patches" patch -p1 < "${PATCH_DIR}/p4runtime-shell-2023-changes.patch" -pip3 install . +dump_python_lib_info "${DEBUG_DUMP_DIR}/099-p4runtime-shell-just-before-pip-install" +uv pip install . +dump_python_lib_info "${DEBUG_DUMP_DIR}/100-p4runtime-shell-just-after-pip-install" pip3 list diff --git a/vm-ubuntu-24.04/patches/behavioral-model-adjust-ubuntu-packges.patch b/vm-ubuntu-24.04/patches/behavioral-model-adjust-ubuntu-packges.patch index f057619d..a74071f5 100644 --- a/vm-ubuntu-24.04/patches/behavioral-model-adjust-ubuntu-packges.patch +++ b/vm-ubuntu-24.04/patches/behavioral-model-adjust-ubuntu-packges.patch @@ -1,5 +1,5 @@ diff --git a/install_deps.sh b/install_deps.sh -index 02dd6d2..7673969 100755 +index 02dd6d2..f1510fe 100755 --- a/install_deps.sh +++ b/install_deps.sh @@ -18,31 +18,20 @@ sudo apt-get install -y \ @@ -35,3 +35,11 @@ index 02dd6d2..7673969 100755 wget tmpdir=`mktemp -d -p .` +@@ -50,7 +39,6 @@ cd $tmpdir + + bash ../ci/install-thrift.sh + bash ../ci/install-nanomsg.sh +-bash ../ci/install-pi.sh + sudo ldconfig + bash ../ci/install-pynng.sh + diff --git a/vm-ubuntu-24.04/patches/behavioral-model-support-venv-2026-apr.patch b/vm-ubuntu-24.04/patches/behavioral-model-support-venv-2026-apr.patch index e7a92c81..ca4cf7e1 100644 --- a/vm-ubuntu-24.04/patches/behavioral-model-support-venv-2026-apr.patch +++ b/vm-ubuntu-24.04/patches/behavioral-model-support-venv-2026-apr.patch @@ -1,8 +1,126 @@ +diff --git a/ci/install-nanomsg.sh b/ci/install-nanomsg.sh +index 675a329..54fcb3d 100644 +--- a/ci/install-nanomsg.sh ++++ b/ci/install-nanomsg.sh +@@ -4,6 +4,34 @@ + # + # SPDX-License-Identifier: Apache-2.0 + ++dump_python_lib_info() { ++ local output_dir=$1 ++ set +e ++ mkdir -p ${output_dir} ++ outf=${output_dir}/dirs.txt ++ echo "All directories named site-packages or dist-packages:" > ${outf} ++ find / -name site-packages -o -name dist-packages | sort >> ${outf} ++ echo "" >> ${outf} ++ echo "ls -la on each such directory:" >> ${outf} ++ for d in $(find / -name site-packages -o -name dist-packages | sort) ++ do ++ echo $d >> ${outf} ++ ls -la $d >> ${outf} ++ done ++ #echo "" >> ${outf} ++ #echo "ls -laR on each such directory:" >> ${outf} ++ for d in $(find / -name site-packages -o -name dist-packages | sort) ++ do ++ outf="${output_dir}/ls-laR-of-$(echo "$d" | tr '/' '-')" ++ cp /dev/null ${outf} ++ #echo $d >> ${outf} ++ ls -laR $d >> ${outf} ++ done ++ set -e ++} ++ ++DEBUG_DUMP_DIR="${HOME}/install-p4dev-dumpdir" ++ + THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + source $THIS_DIR/common.sh + +@@ -22,5 +50,7 @@ cd build + # some reason ldconfig cannot find it + cmake .. -DCMAKE_INSTALL_PREFIX=/usr + cmake --build . ++dump_python_lib_info "${DEBUG_DUMP_DIR}/033b-behavioral-model-just-before-nanomsg-cmake-install" + sudo cmake --build . --target install ++dump_python_lib_info "${DEBUG_DUMP_DIR}/033c-behavioral-model-just-after-nanomsg-cmake-install" + cd .. +diff --git a/ci/install-pynng.sh b/ci/install-pynng.sh +index 6abcd8e..3b4ae9f 100644 +--- a/ci/install-pynng.sh ++++ b/ci/install-pynng.sh +@@ -4,5 +4,34 @@ + # + # SPDX-License-Identifier: Apache-2.0 + ++dump_python_lib_info() { ++ local output_dir=$1 ++ set +e ++ mkdir -p ${output_dir} ++ outf=${output_dir}/dirs.txt ++ echo "All directories named site-packages or dist-packages:" > ${outf} ++ find / -name site-packages -o -name dist-packages | sort >> ${outf} ++ echo "" >> ${outf} ++ echo "ls -la on each such directory:" >> ${outf} ++ for d in $(find / -name site-packages -o -name dist-packages | sort) ++ do ++ echo $d >> ${outf} ++ ls -la $d >> ${outf} ++ done ++ #echo "" >> ${outf} ++ #echo "ls -laR on each such directory:" >> ${outf} ++ for d in $(find / -name site-packages -o -name dist-packages | sort) ++ do ++ outf="${output_dir}/ls-laR-of-$(echo "$d" | tr '/' '-')" ++ cp /dev/null ${outf} ++ #echo $d >> ${outf} ++ ls -laR $d >> ${outf} ++ done ++ set -e ++} ++ ++DEBUG_DUMP_DIR="${HOME}/install-p4dev-dumpdir" ++ + set -e +-python3 -m pip install --upgrade pynng==0.9.0 ++uv pip install pynng==0.9.0 ++dump_python_lib_info "${DEBUG_DUMP_DIR}/034-behavioral-model-just-after-pip-install-pynng" diff --git a/ci/install-thrift.sh b/ci/install-thrift.sh -index 1316476..ea6203c 100644 +index 587e9d3..088569d 100644 --- a/ci/install-thrift.sh +++ b/ci/install-thrift.sh -@@ -3,22 +3,18 @@ +@@ -4,25 +4,51 @@ + # + # SPDX-License-Identifier: Apache-2.0 + ++dump_python_lib_info() { ++ local output_dir=$1 ++ set +e ++ mkdir -p ${output_dir} ++ outf=${output_dir}/dirs.txt ++ echo "All directories named site-packages or dist-packages:" > ${outf} ++ find / -name site-packages -o -name dist-packages | sort >> ${outf} ++ echo "" >> ${outf} ++ echo "ls -la on each such directory:" >> ${outf} ++ for d in $(find / -name site-packages -o -name dist-packages | sort) ++ do ++ echo $d >> ${outf} ++ ls -la $d >> ${outf} ++ done ++ #echo "" >> ${outf} ++ #echo "ls -laR on each such directory:" >> ${outf} ++ for d in $(find / -name site-packages -o -name dist-packages | sort) ++ do ++ outf="${output_dir}/ls-laR-of-$(echo "$d" | tr '/' '-')" ++ cp /dev/null ${outf} ++ #echo $d >> ${outf} ++ ls -laR $d >> ${outf} ++ done ++ set -e ++} ++ ++DEBUG_DUMP_DIR="${HOME}/install-p4dev-dumpdir" ++ THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) source $THIS_DIR/common.sh @@ -29,5 +147,7 @@ index 1316476..ea6203c 100644 -cd lib/py -sudo python3 setup.py install -cd ../../.. ++dump_python_lib_info "${DEBUG_DUMP_DIR}/032-behavioral-model-just-before-pip-install-thrift" +cd .. -+pip install thrift==0.16.0 ++uv pip install thrift==0.16.0 ++dump_python_lib_info "${DEBUG_DUMP_DIR}/033-behavioral-model-just-after-pip-install-thrift" diff --git a/vm-ubuntu-24.04/patches/mininet-patch-for-2024-sep-enable-venv.patch b/vm-ubuntu-24.04/patches/mininet-patch-for-2024-sep-enable-venv.patch index cc084f84..f7c79837 100644 --- a/vm-ubuntu-24.04/patches/mininet-patch-for-2024-sep-enable-venv.patch +++ b/vm-ubuntu-24.04/patches/mininet-patch-for-2024-sep-enable-venv.patch @@ -1,5 +1,5 @@ diff --git a/util/install.sh b/util/install.sh -index bf4492d..998cc22 100755 +index bf4492d..1b79b01 100755 --- a/util/install.sh +++ b/util/install.sh @@ -5,6 +5,7 @@ @@ -18,7 +18,7 @@ index bf4492d..998cc22 100755 # Starting around 20.04, installing pyflakes instead of pyflakes3 # causes Python 2 to be installed, which is exactly NOT what we want. if [ "$DIST" = "Ubuntu" -a `expr $RELEASE '>=' 20.04` = "1" ]; then -@@ -187,10 +189,17 @@ function mn_deps { +@@ -187,23 +189,19 @@ function mn_deps { pf=pyflakes3 pep8=python3-pep8 fi @@ -35,21 +35,35 @@ index bf4492d..998cc22 100755 net-tools ${PYPKG}-tk + $install --no-install-recommends $pf $pep8 - # Install pip - $install ${PYPKG}-pip || $install ${PYPKG}-pip-whl -@@ -211,7 +220,10 @@ function mn_deps { +- # Install pip +- $install ${PYPKG}-pip || $install ${PYPKG}-pip-whl +- if ! ${PYTHON} -m pip -V; then +- if [ $PYTHON_VERSION == 2 ]; then +- wget https://bootstrap.pypa.io/pip/2.7/get-pip.py +- else +- wget https://bootstrap.pypa.io/get-pip.py +- fi +- sudo ${PYTHON} get-pip.py +- rm get-pip.py +- fi +- ${python} -m pip install pexpect ++ uv pip install pexpect + $install iproute2 || $install iproute + $install cgroup-tools || $install cgroup-bin + $install cgroupfs-mount +@@ -211,7 +209,10 @@ function mn_deps { echo "Installing Mininet core" pushd $MININET_DIR/mininet - sudo PYTHON=${PYTHON} make install + sudo PYTHON=${PYTHON} make install-mnexec + sudo PYTHON=${PYTHON} make install-manpages -+ ${PYTHON} -m pip uninstall -y mininet || true -+ ${PYTHON} -m pip install -e . --no-binary :all: ++ uv pip uninstall -y mininet || true ++ uv pip install -e . --no-binary :all: popd } -@@ -303,7 +315,7 @@ function install_wireshark { +@@ -303,7 +304,7 @@ function install_wireshark { if ! which wireshark; then echo "Installing Wireshark" if [ "$DIST" = "Fedora" -o "$DIST" = "RedHatEnterpriseServer" ]; then