1#!/bin/bash 2 3usage () { 4 echo "USAGE: ./sync-kernel.sh <libbpf-repo> <kernel-repo> <bpf-branch>" 5 echo "" 6 echo "Set BPF_NEXT_BASELINE to override bpf-next tree commit, otherwise read from <libbpf-repo>/CHECKPOINT-COMMIT." 7 echo "Set BPF_BASELINE to override bpf tree commit, otherwise read from <libbpf-repo>/BPF-CHECKPOINT-COMMIT." 8 echo "Set MANUAL_MODE to 1 to manually control every cherry-picked commits." 9 exit 1 10} 11 12set -eu 13 14LIBBPF_REPO=${1-""} 15LINUX_REPO=${2-""} 16BPF_BRANCH=${3-""} 17BASELINE_COMMIT=${BPF_NEXT_BASELINE:-$(cat ${LIBBPF_REPO}/CHECKPOINT-COMMIT)} 18BPF_BASELINE_COMMIT=${BPF_BASELINE:-$(cat ${LIBBPF_REPO}/BPF-CHECKPOINT-COMMIT)} 19 20if [ -z "${LIBBPF_REPO}" ] || [ -z "${LINUX_REPO}" ]; then 21 echo "Error: libbpf or linux repos are not specified" 22 usage 23fi 24if [ -z "${BPF_BRANCH}" ]; then 25 echo "Error: linux's bpf tree branch is not specified" 26 usage 27fi 28if [ -z "${BASELINE_COMMIT}" ] || [ -z "${BPF_BASELINE_COMMIT}" ]; then 29 echo "Error: bpf or bpf-next baseline commits are not provided" 30 usage 31fi 32 33SUFFIX=$(date --utc +%Y-%m-%dT%H-%M-%S.%3NZ) 34WORKDIR=$(pwd) 35TMP_DIR=$(mktemp -d) 36 37trap "cd ${WORKDIR}; exit" INT TERM EXIT 38 39declare -A PATH_MAP 40PATH_MAP=( \ 41 [tools/lib/bpf]=src \ 42 [tools/include/uapi/linux/bpf_common.h]=include/uapi/linux/bpf_common.h \ 43 [tools/include/uapi/linux/bpf.h]=include/uapi/linux/bpf.h \ 44 [tools/include/uapi/linux/btf.h]=include/uapi/linux/btf.h \ 45 [tools/include/uapi/linux/fcntl.h]=include/uapi/linux/fcntl.h \ 46 [tools/include/uapi/linux/openat2.h]=include/uapi/linux/openat2.h \ 47 [tools/include/uapi/linux/if_link.h]=include/uapi/linux/if_link.h \ 48 [tools/include/uapi/linux/if_xdp.h]=include/uapi/linux/if_xdp.h \ 49 [tools/include/uapi/linux/netdev.h]=include/uapi/linux/netdev.h \ 50 [tools/include/uapi/linux/netlink.h]=include/uapi/linux/netlink.h \ 51 [tools/include/uapi/linux/pkt_cls.h]=include/uapi/linux/pkt_cls.h \ 52 [tools/include/uapi/linux/pkt_sched.h]=include/uapi/linux/pkt_sched.h \ 53 [include/uapi/linux/perf_event.h]=include/uapi/linux/perf_event.h \ 54 [Documentation/bpf/libbpf]=docs \ 55) 56 57LIBBPF_PATHS=("${!PATH_MAP[@]}" ":^tools/lib/bpf/Makefile" ":^tools/lib/bpf/Build" ":^tools/lib/bpf/.gitignore" ":^tools/include/tools/libc_compat.h") 58LIBBPF_VIEW_PATHS=("${PATH_MAP[@]}") 59LIBBPF_VIEW_EXCLUDE_REGEX='^src/(Makefile|Build|test_libbpf\.c|bpf_helper_defs\.h|\.gitignore)$|^docs/(\.gitignore|api\.rst|conf\.py)$|^docs/sphinx/.*' 60LINUX_VIEW_EXCLUDE_REGEX='^include/tools/libc_compat.h$' 61 62LIBBPF_TREE_FILTER="mkdir -p __libbpf/include/uapi/linux __libbpf/include/tools && "$'\\\n' 63for p in "${!PATH_MAP[@]}"; do 64 LIBBPF_TREE_FILTER+="git mv -kf ${p} __libbpf/${PATH_MAP[${p}]} && "$'\\\n' 65done 66LIBBPF_TREE_FILTER+="git rm --ignore-unmatch -f __libbpf/src/{Makefile,Build,test_libbpf.c,.gitignore} >/dev/null" 67 68cd_to() 69{ 70 cd ${WORKDIR} && cd "$1" 71} 72 73# Output brief single-line commit description 74# $1 - commit ref 75commit_desc() 76{ 77 git log -n1 --pretty='%h ("%s")' $1 78} 79 80# Create commit single-line signature, which consists of: 81# - full commit subject 82# - author date in ISO8601 format 83# - full commit body with newlines replaced with vertical bars (|) 84# - shortstat appended at the end 85# The idea is that this single-line signature is good enough to make final 86# decision about whether two commits are the same, across different repos. 87# $1 - commit ref 88# $2 - paths filter 89commit_signature() 90{ 91 local ref=$1 92 shift 93 git show --pretty='("%s")|%aI|%b' --shortstat $ref -- "${@-.}" | tr '\n' '|' 94} 95 96# Cherry-pick commits touching libbpf-related files 97# $1 - baseline_tag 98# $2 - tip_tag 99cherry_pick_commits() 100{ 101 local manual_mode=${MANUAL_MODE:-0} 102 local baseline_tag=$1 103 local tip_tag=$2 104 local new_commits 105 local signature 106 local should_skip 107 local synced_cnt 108 local manual_check 109 local libbpf_conflict_cnt 110 local desc 111 112 new_commits=$(git rev-list --no-merges --topo-order --reverse ${baseline_tag}..${tip_tag} -- "${LIBBPF_PATHS[@]}") 113 for new_commit in ${new_commits}; do 114 desc="$(commit_desc ${new_commit})" 115 signature="$(commit_signature ${new_commit} "${LIBBPF_PATHS[@]}")" 116 synced_cnt=$(grep -F "${signature}" ${TMP_DIR}/libbpf_commits.txt | wc -l) 117 manual_check=0 118 if ((${synced_cnt} > 0)); then 119 # commit with the same subject is already in libbpf, but it's 120 # not 100% the same commit, so check with user 121 echo "Commit '${desc}' is synced into libbpf as:" 122 grep -F "${signature}" ${TMP_DIR}/libbpf_commits.txt | \ 123 cut -d'|' -f1 | sed -e 's/^/- /' 124 if ((${manual_mode} != 1 && ${synced_cnt} == 1)); then 125 echo "Skipping '${desc}' due to unique match..." 126 continue 127 fi 128 if ((${synced_cnt} > 1)); then 129 echo "'${desc} matches multiple commits, please, double-check!" 130 manual_check=1 131 fi 132 fi 133 if ((${manual_mode} == 1 || ${manual_check} == 1)); then 134 read -p "Do you want to skip '${desc}'? [y/N]: " should_skip 135 case "${should_skip}" in 136 "y" | "Y") 137 echo "Skipping '${desc}'..." 138 continue 139 ;; 140 esac 141 fi 142 # commit hasn't been synced into libbpf yet 143 echo "Picking '${desc}'..." 144 if ! git cherry-pick ${new_commit} &>/dev/null; then 145 echo "Warning! Cherry-picking '${desc} failed, checking if it's non-libbpf files causing problems..." 146 libbpf_conflict_cnt=$(git diff --name-only --diff-filter=U -- "${LIBBPF_PATHS[@]}" | wc -l) 147 conflict_cnt=$(git diff --name-only | wc -l) 148 prompt_resolution=1 149 150 if ((${libbpf_conflict_cnt} == 0)); then 151 echo "Looks like only non-libbpf files have conflicts, ignoring..." 152 if ((${conflict_cnt} == 0)); then 153 echo "Empty cherry-pick, skipping it..." 154 git cherry-pick --abort 155 continue 156 fi 157 158 git add . 159 # GIT_EDITOR=true to avoid editor popping up to edit commit message 160 if ! GIT_EDITOR=true git cherry-pick --continue &>/dev/null; then 161 echo "Error! That still failed! Please resolve manually." 162 else 163 echo "Success! All cherry-pick conflicts were resolved for '${desc}'!" 164 prompt_resolution=0 165 fi 166 fi 167 168 if ((${prompt_resolution} == 1)); then 169 read -p "Error! Cherry-picking '${desc}' failed, please fix manually and press <return> to proceed..." 170 fi 171 fi 172 # Append signature of just cherry-picked commit to avoid 173 # potentially cherry-picking the same commit twice later when 174 # processing bpf tree commits. At this point we don't know yet 175 # the final commit sha in libbpf repo, so we record Linux SHA 176 # instead as LINUX_<sha>. 177 echo LINUX_$(git log --pretty='%h' -n1) "${signature}" >> ${TMP_DIR}/libbpf_commits.txt 178 done 179} 180 181cleanup() 182{ 183 echo "Cleaning up..." 184 rm -r ${TMP_DIR} 185 cd_to ${LINUX_REPO} 186 git checkout ${TIP_SYM_REF} 187 git branch -D ${BASELINE_TAG} ${TIP_TAG} ${BPF_BASELINE_TAG} ${BPF_TIP_TAG} \ 188 ${SQUASH_BASE_TAG} ${SQUASH_TIP_TAG} ${VIEW_TAG} || true 189 190 cd_to . 191 echo "DONE." 192} 193 194 195cd_to ${LIBBPF_REPO} 196GITHUB_ABS_DIR=$(pwd) 197echo "Dumping existing libbpf commit signatures..." 198for h in $(git log --pretty='%h' -n500); do 199 echo $h "$(commit_signature $h)" >> ${TMP_DIR}/libbpf_commits.txt 200done 201 202# Use current kernel repo HEAD as a source of patches 203cd_to ${LINUX_REPO} 204LINUX_ABS_DIR=$(pwd) 205TIP_SYM_REF=$(git symbolic-ref -q --short HEAD || git rev-parse HEAD) 206TIP_COMMIT=$(git rev-parse HEAD) 207BPF_TIP_COMMIT=$(git rev-parse ${BPF_BRANCH}) 208BASELINE_TAG=libbpf-baseline-${SUFFIX} 209TIP_TAG=libbpf-tip-${SUFFIX} 210BPF_BASELINE_TAG=libbpf-bpf-baseline-${SUFFIX} 211BPF_TIP_TAG=libbpf-bpf-tip-${SUFFIX} 212VIEW_TAG=libbpf-view-${SUFFIX} 213LIBBPF_SYNC_TAG=libbpf-sync-${SUFFIX} 214 215# Squash state of kernel repo at baseline into single commit 216SQUASH_BASE_TAG=libbpf-squash-base-${SUFFIX} 217SQUASH_TIP_TAG=libbpf-squash-tip-${SUFFIX} 218SQUASH_COMMIT=$(git commit-tree ${BASELINE_COMMIT}^{tree} -m "BASELINE SQUASH ${BASELINE_COMMIT}") 219 220echo "WORKDIR: ${WORKDIR}" 221echo "LINUX REPO: ${LINUX_REPO}" 222echo "LIBBPF REPO: ${LIBBPF_REPO}" 223echo "TEMP DIR: ${TMP_DIR}" 224echo "SUFFIX: ${SUFFIX}" 225echo "BASE COMMIT: '$(commit_desc ${BASELINE_COMMIT})'" 226echo "TIP COMMIT: '$(commit_desc ${TIP_COMMIT})'" 227echo "BPF BASE COMMIT: '$(commit_desc ${BPF_BASELINE_COMMIT})'" 228echo "BPF TIP COMMIT: '$(commit_desc ${BPF_TIP_COMMIT})'" 229echo "SQUASH COMMIT: ${SQUASH_COMMIT}" 230echo "BASELINE TAG: ${BASELINE_TAG}" 231echo "TIP TAG: ${TIP_TAG}" 232echo "BPF BASELINE TAG: ${BPF_BASELINE_TAG}" 233echo "BPF TIP TAG: ${BPF_TIP_TAG}" 234echo "SQUASH BASE TAG: ${SQUASH_BASE_TAG}" 235echo "SQUASH TIP TAG: ${SQUASH_TIP_TAG}" 236echo "VIEW TAG: ${VIEW_TAG}" 237echo "LIBBPF SYNC TAG: ${LIBBPF_SYNC_TAG}" 238echo "PATCHES: ${TMP_DIR}/patches" 239 240git branch ${BASELINE_TAG} ${BASELINE_COMMIT} 241git branch ${TIP_TAG} ${TIP_COMMIT} 242git branch ${BPF_BASELINE_TAG} ${BPF_BASELINE_COMMIT} 243git branch ${BPF_TIP_TAG} ${BPF_TIP_COMMIT} 244git branch ${SQUASH_BASE_TAG} ${SQUASH_COMMIT} 245git checkout -b ${SQUASH_TIP_TAG} ${SQUASH_COMMIT} 246 247# Cherry-pick new commits onto squashed baseline commit 248cherry_pick_commits ${BASELINE_TAG} ${TIP_TAG} 249cherry_pick_commits ${BPF_BASELINE_TAG} ${BPF_TIP_TAG} 250 251# Move all libbpf files into __libbpf directory. 252FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --tree-filter "${LIBBPF_TREE_FILTER}" ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG} 253# Make __libbpf a new root directory 254FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch --prune-empty -f --subdirectory-filter __libbpf ${SQUASH_TIP_TAG} ${SQUASH_BASE_TAG} 255 256# If there are no new commits with libbpf-related changes, bail out 257COMMIT_CNT=$(git rev-list --count ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG}) 258if ((${COMMIT_CNT} <= 0)); then 259 echo "No new changes to apply, we are done!" 260 cleanup 261 exit 2 262fi 263 264# Exclude baseline commit and generate nice cover letter with summary 265git format-patch --no-signature ${SQUASH_BASE_TAG}..${SQUASH_TIP_TAG} --cover-letter -o ${TMP_DIR}/patches 266 267# Now is time to re-apply libbpf-related linux patches to libbpf repo 268cd_to ${LIBBPF_REPO} 269git checkout -b ${LIBBPF_SYNC_TAG} 270 271for patch in $(ls -1 ${TMP_DIR}/patches | tail -n +2); do 272 if ! git am -3 --committer-date-is-author-date "${TMP_DIR}/patches/${patch}"; then 273 if ! patch -p1 --merge < "${TMP_DIR}/patches/${patch}"; then 274 read -p "Applying ${TMP_DIR}/patches/${patch} failed, please resolve manually and press <return> to proceed..." 275 fi 276 git am --continue 277 fi 278done 279 280# Generate bpf_helper_defs.h and commit, if anything changed 281# restore Linux tip to use bpf_doc.py 282cd_to ${LINUX_REPO} 283git checkout ${TIP_TAG} 284# re-generate bpf_helper_defs.h 285cd_to ${LIBBPF_REPO} 286"${LINUX_ABS_DIR}/scripts/bpf_doc.py" --header \ 287 --file include/uapi/linux/bpf.h > src/bpf_helper_defs.h 288# if anything changed, commit it 289helpers_changes=$(git status --porcelain src/bpf_helper_defs.h | wc -l) 290if ((${helpers_changes} == 1)); then 291 git add src/bpf_helper_defs.h 292 git commit -s -m "sync: auto-generate latest BPF helpers 293 294Latest changes to BPF helper definitions. 295" -- src/bpf_helper_defs.h 296fi 297 298# Use generated cover-letter as a template for "sync commit" with 299# baseline and checkpoint commits from kernel repo (and leave summary 300# from cover letter intact, of course) 301echo ${TIP_COMMIT} > CHECKPOINT-COMMIT && \ 302echo ${BPF_TIP_COMMIT} > BPF-CHECKPOINT-COMMIT && \ 303git add CHECKPOINT-COMMIT && \ 304git add BPF-CHECKPOINT-COMMIT && \ 305awk '/\*\*\* BLURB HERE \*\*\*/ {p=1} p' ${TMP_DIR}/patches/0000-cover-letter.patch | \ 306sed "s/\*\*\* BLURB HERE \*\*\*/\ 307sync: latest libbpf changes from kernel\n\ 308\n\ 309Syncing latest libbpf commits from kernel repository.\n\ 310Baseline bpf-next commit: ${BASELINE_COMMIT}\n\ 311Checkpoint bpf-next commit: ${TIP_COMMIT}\n\ 312Baseline bpf commit: ${BPF_BASELINE_COMMIT}\n\ 313Checkpoint bpf commit: ${BPF_TIP_COMMIT}/" | \ 314git commit -s --file=- 315 316echo "SUCCESS! ${COMMIT_CNT} commits synced." 317 318echo "Verifying Linux's and Github's libbpf state" 319 320cd_to ${LINUX_REPO} 321git checkout -b ${VIEW_TAG} ${TIP_COMMIT} 322FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --tree-filter "${LIBBPF_TREE_FILTER}" ${VIEW_TAG}^..${VIEW_TAG} 323FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch -f --subdirectory-filter __libbpf ${VIEW_TAG}^..${VIEW_TAG} 324git ls-files -- "${LIBBPF_VIEW_PATHS[@]}" | grep -v -E "${LINUX_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/linux-view.ls 325 326cd_to ${LIBBPF_REPO} 327git ls-files -- "${LIBBPF_VIEW_PATHS[@]}" | grep -v -E "${LIBBPF_VIEW_EXCLUDE_REGEX}" > ${TMP_DIR}/github-view.ls 328 329echo "Comparing list of files..." 330diff -u ${TMP_DIR}/linux-view.ls ${TMP_DIR}/github-view.ls 331echo "Comparing file contents..." 332CONSISTENT=1 333for F in $(cat ${TMP_DIR}/linux-view.ls); do 334 if ! diff -u "${LINUX_ABS_DIR}/${F}" "${GITHUB_ABS_DIR}/${F}"; then 335 echo "${LINUX_ABS_DIR}/${F} and ${GITHUB_ABS_DIR}/${F} are different!" 336 CONSISTENT=0 337 fi 338done 339if ((${CONSISTENT} == 1)); then 340 echo "Great! Content is identical!" 341else 342 ignore_inconsistency=n 343 echo "Unfortunately, there are some inconsistencies, please double check." 344 read -p "Does everything look good? [y/N]: " ignore_inconsistency 345 case "${ignore_inconsistency}" in 346 "y" | "Y") 347 echo "Ok, proceeding..." 348 ;; 349 *) 350 echo "Oops, exiting with error..." 351 exit 4 352 esac 353fi 354 355cleanup 356