mirror of https://git.FreeBSD.org/ports.git
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
507 lines
16 KiB
507 lines
16 KiB
#!/usr/libexec/flua |
|
|
|
--[[ |
|
SPDX-License-Identifier: BSD-2-Clause-FreeBSD |
|
|
|
Copyright (c) 2022 Stefan Esser <se@FreeBSD.org> |
|
|
|
Generate a list of existing and required CONFLICTS_INSTALL lines |
|
for all ports (limited to ports for which official packages are |
|
provided). |
|
|
|
This script depends on the ports-mgmt/pkg-provides port for the list |
|
of files installed by all pre-built packages for the architecture |
|
the script is run on. |
|
|
|
The script generates a list of ports by running "pkg provides ." and |
|
a mapping from package base name to origin via "pkg rquery '%n %o'". |
|
|
|
The existing CONFLICTS and CONFLICTS_INSTALL definitions are fetched |
|
by "make -C $origin -V CONFLICTS -V CONFLICTS_INSTALL". This list is |
|
only representative for the options configured for each port (i.e. |
|
if non-default options have been selected and registered, these may |
|
lead to a non-default list of conflicts). |
|
|
|
The script detects files used by more than one port, than lists by |
|
origin the existing definition and the list of package base names |
|
that have been detected to cause install conflicts followed by the |
|
list of duplicate files separated by a hash character "#". |
|
|
|
This script uses the "hidden" LUA interpreter in the FreeBSD base |
|
systems and does not need any port except "pkg-provides" to be run. |
|
|
|
The run-time on my system checking the ~32000 packages available |
|
for -CURRENT on amd64 is less than 250 seconds. |
|
|
|
Example output: |
|
|
|
# Port: games/sol |
|
# Files: bin/sol |
|
# < aisleriot gnome-games |
|
# > aisleriot |
|
portedit merge -ie 'CONFLICTS_INSTALL=aisleriot # bin/sol' /usr/ports/games/sol |
|
|
|
The output is per port (for all flavors of the port, if applicable), |
|
gives examples of conflicting files (mostly to understand whether |
|
different versions of a port could co-exist), the current CONFLICTS |
|
and CONFLICTS_INSTALL entries merged, and a suggested new entry. |
|
This information is followed by a portedit command line that should |
|
do the right thing for simple cases, but the result should always |
|
be checked before the resulting Makefile is committed. |
|
--]] |
|
|
|
require "lfs" |
|
|
|
------------------------------------------------------------------- |
|
local file_pattern = "." |
|
local database = "/var/db/pkg/provides/provides.db" |
|
local max_age = 1 * 24 * 3600 -- maximum age of database file in seconds |
|
|
|
------------------------------------------------------------------- |
|
local function table_sorted_keys(t) |
|
local result = {} |
|
for k, _ in pairs(t) do |
|
result[#result + 1] = k |
|
end |
|
table.sort(result) |
|
return result |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local function table_sort_uniq(t) |
|
local result = {} |
|
if t then |
|
local last |
|
table.sort(t) |
|
for _, entry in ipairs(t) do |
|
if entry ~= last then |
|
last = entry |
|
result[#result + 1] = entry |
|
end |
|
end |
|
end |
|
return result |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local function fnmatch(name, pattern) |
|
local function fnsubst(s) |
|
s = string.gsub(s, "%%", "%%%%") |
|
s = string.gsub(s, "%+", "%%+") |
|
s = string.gsub(s, "%-", "%%-") |
|
s = string.gsub(s, "%.", "%%.") |
|
s = string.gsub(s, "%?", ".") |
|
s = string.gsub(s, "%*", ".*") |
|
return s |
|
end |
|
local rexpr = "" |
|
local left, middle, right |
|
while true do |
|
left, middle, right = string.match(pattern, "([^[]*)(%[[^]]+%])(.*)") |
|
if not left then |
|
break |
|
end |
|
rexpr = rexpr .. fnsubst(left) .. middle |
|
pattern = right |
|
end |
|
rexpr = "^" .. rexpr .. fnsubst(pattern) .. "$" |
|
return string.find(name, rexpr) |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local function fetch_pkgs_origins() |
|
local pkgs = {} |
|
local pipe = io.popen("pkg rquery '%n %o'") |
|
for line in pipe:lines() do |
|
local pkgbase, origin = string.match(line, "(%S+) (%S+)") |
|
pkgs[origin] = pkgbase |
|
end |
|
pipe:close() |
|
pipe = io.popen("pkg rquery '%n %o %At %Av'") |
|
for line in pipe:lines() do |
|
local pkgbase, origin, tag, value = string.match(line, "(%S+) (%S+) (%S+) (%S+)") |
|
if tag == "flavor" then |
|
pkgs[origin] = nil |
|
pkgs[origin .. "@" .. value] = pkgbase |
|
end |
|
end |
|
pipe:close() |
|
return pkgs |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local BAD_FILE_PATTERN = { |
|
"^[^/]+$", |
|
"^lib/python[%d%.]+/site%-packages/examples/[^/]+$", |
|
"^lib/python[%d%.]+/site%-packages/samples/[^/]+$", |
|
"^lib/python[%d%.]+/site%-packages/test/[^/]+$", |
|
"^lib/python[%d%.]+/site%-packages/test_app/[^/]+$", |
|
"^lib/python[%d%.]+/site%-packages/tests/[^/]+$", |
|
"^lib/python[%d%.]+/site%-packages/tests/unit/[^/]+$", |
|
} |
|
|
|
local BAD_FILE_PKGS = {} |
|
|
|
local function check_bad_file(pkgbase, file) |
|
for _, pattern in ipairs(BAD_FILE_PATTERN) do |
|
if string.match(file, pattern) then |
|
BAD_FILE_PKGS[pkgbase] = BAD_FILE_PKGS[pkgbase] or {} |
|
table.insert(BAD_FILE_PKGS[pkgbase], file) |
|
break |
|
end |
|
end |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local function read_files(pattern) |
|
local files_table = {} |
|
local now = os.time() |
|
local modification_time = lfs.attributes(database, "modification") |
|
if not modification_time then |
|
print("# Aborting: package file database " .. database .. " does not exist.") |
|
print("# Install the 'pkg-provides' package and add it as a module to 'pkg.conf'.") |
|
print("# Then fetch the database with 'pkg update' or 'pkg provides -u'.") |
|
os.exit(1) |
|
end |
|
if now - modification_time > max_age then |
|
print("# Aborting: package file database " .. database) |
|
print("# is older than " .. max_age .. " seconds.") |
|
print("# Use 'pkg provides -u' to update the database.") |
|
os.exit(2) |
|
end |
|
local pipe = io.popen("locate -d " .. database .. " " .. pattern) |
|
if pipe then |
|
for line in pipe:lines() do |
|
local pkgbase, file = string.match(line, "([^*]+)%*([^*]+)") |
|
if file:sub(1, 11) == "/usr/local/" then |
|
file = file:sub(12) |
|
end |
|
check_bad_file(pkgbase, file) |
|
local t = files_table[file] or {} |
|
t[#t + 1] = pkgbase |
|
files_table[file] = t |
|
end |
|
pipe:close() |
|
end |
|
return files_table |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local DUPLICATE_FILE = {} |
|
|
|
local function fetch_pkg_pairs(pattern) |
|
local pkg_pairs = {} |
|
for file, pkgbases in pairs(read_files(pattern)) do |
|
if #pkgbases >= 2 then |
|
DUPLICATE_FILE[file] = true |
|
for i = 1, #pkgbases -1 do |
|
local pkg_i = pkgbases[i] |
|
for j = i + 1, #pkgbases do |
|
local pkg_j = pkgbases[j] |
|
if pkg_i ~= pkg_j then |
|
local p1 = pkg_pairs[pkg_i] or {} |
|
local p2 = p1[pkg_j] or {} |
|
p2[#p2 + 1] = file |
|
p1[pkg_j] = p2 |
|
pkg_pairs[pkg_i] = p1 |
|
end |
|
end |
|
end |
|
end |
|
end |
|
return pkg_pairs |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local function conflicts_delta(old, new) |
|
local old_seen = {} |
|
local changed |
|
for i = 1, #new do |
|
local matched |
|
for j = 1, #old do |
|
if new[i] == old[j] or fnmatch(new[i], old[j]) then |
|
new[i] = old[j] |
|
old_seen[j] = true |
|
matched = true |
|
break |
|
end |
|
end |
|
changed = changed or not matched |
|
end |
|
if not changed then |
|
for j = 1, #old do |
|
if not old_seen[j] then |
|
changed = true |
|
break |
|
end |
|
end |
|
end |
|
if changed then |
|
return table_sort_uniq(new) |
|
end |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local function fetch_port_conflicts(origin) |
|
local dir, flavor = origin:match("([^@]+)@?(.*)") |
|
if flavor ~= "" then |
|
flavor = " FLAVOR=" .. flavor |
|
end |
|
local seen = {} |
|
local portdir = "/usr/ports/" .. dir |
|
local pipe = io.popen("make -C " .. portdir .. flavor .. " -V CONFLICTS -V CONFLICTS_INSTALL 2>/dev/null") |
|
for line in pipe:lines() do |
|
for word in line:gmatch("(%S+)%s?") do |
|
seen[word] = true |
|
end |
|
end |
|
pipe:close() |
|
return table_sorted_keys(seen) |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local function conflicting_pkgs(conflicting) |
|
local pkgs = {} |
|
for origin, pkgbase in pairs(fetch_pkgs_origins()) do |
|
if conflicting[pkgbase] then |
|
pkgs[origin] = pkgbase |
|
end |
|
end |
|
return pkgs |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local function collect_conflicts(pkg_pairs) |
|
local pkgs = {} |
|
for pkg_i, p1 in pairs(pkg_pairs) do |
|
for pkg_j, _ in pairs(p1) do |
|
pkgs[pkg_i] = pkgs[pkg_i] or {} |
|
table.insert(pkgs[pkg_i], pkg_j) |
|
pkgs[pkg_j] = pkgs[pkg_j] or {} |
|
table.insert(pkgs[pkg_j], pkg_i) |
|
end |
|
end |
|
return pkgs |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local function split_origins(origin_list) |
|
local port_list = {} |
|
local flavors = {} |
|
local last_port |
|
for _, origin in ipairs(origin_list) do |
|
local port, flavor = string.match(origin, "([^@]+)@?(.*)") |
|
if port ~= last_port then |
|
port_list[#port_list + 1] = port |
|
if flavor ~= "" then |
|
flavors[port] = {flavor} |
|
end |
|
else |
|
table.insert(flavors[port], flavor) |
|
end |
|
last_port = port |
|
end |
|
return port_list, flavors |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local PKG_PAIR_FILES = fetch_pkg_pairs(file_pattern) |
|
local CONFLICT_PKGS = collect_conflicts(PKG_PAIR_FILES) |
|
local PKGBASE = conflicting_pkgs(CONFLICT_PKGS) |
|
local ORIGIN_LIST = table_sorted_keys(PKGBASE) |
|
local PORT_LIST, FLAVORS = split_origins(ORIGIN_LIST) |
|
|
|
local function conflicting_files(pkg_i, pkgs) |
|
local files = {} |
|
local all_files = {} |
|
local f |
|
local p1 = PKG_PAIR_FILES[pkg_i] |
|
if p1 then |
|
for _, pkg_j in ipairs(pkgs) do |
|
f = p1[pkg_j] |
|
if f then |
|
table.sort(f) |
|
files[#files + 1] = f[1] |
|
table.move(f, 1, #f, #all_files + 1, all_files) |
|
end |
|
end |
|
end |
|
for _, pkg_j in ipairs(pkgs) do |
|
p1 = PKG_PAIR_FILES[pkg_j] |
|
f = p1 and p1[pkg_i] |
|
if f then |
|
table.sort(f) |
|
files[#files + 1] = f[1] |
|
table.move(f, 1, #f, #all_files + 1, all_files) |
|
end |
|
end |
|
return table_sort_uniq(files), table_sort_uniq(all_files) |
|
end |
|
|
|
--------------------------------------------------------------------- |
|
local version_pattern = { |
|
"^lib/python%d%.%d/", |
|
"^share/py3%d%d%?-", |
|
"^share/%a+/py3%d%d%?-", |
|
"^lib/lua/%d%.%d/", |
|
"^share/lua/%d%.%d/", |
|
"^lib/perl5/[%d%.]+/", |
|
"^lib/perl5/site_perl/mach/[%d%.]+/", |
|
"^lib/ruby/gems/%d%.%d/", |
|
"^lib/ruby/site_ruby/%d%.%d/", |
|
} |
|
|
|
local function generalize_patterns(pkgs, files) |
|
local function match_any(str, pattern_array) |
|
for _, pattern in ipairs(pattern_array) do |
|
if string.match(str, pattern) then |
|
return true |
|
end |
|
end |
|
return false |
|
end |
|
local function unversioned_files() |
|
for i = 1, #files do |
|
if not match_any(files[i], version_pattern) then |
|
return true |
|
end |
|
end |
|
return false |
|
end |
|
local function pkg_wildcards(from, ...) |
|
local to_list = {...} |
|
local result = {} |
|
for i = 1, #pkgs do |
|
local orig_pkg = pkgs[i] |
|
for _, to in ipairs(to_list) do |
|
result[string.gsub(orig_pkg, from, to)] = true |
|
end |
|
end |
|
pkgs = table_sorted_keys(result) |
|
end |
|
local pkg_pfx_php = "php[0-9][0-9]-" |
|
local pkg_sfx_php = "-php[0-9][0-9]" |
|
local pkg_pfx_python2 |
|
local pkg_sfx_python2 |
|
local pkg_pfx_python3 |
|
local pkg_sfx_python3 |
|
local pkg_pfx_lua |
|
local pkg_pfx_ruby |
|
pkgs = table_sort_uniq(pkgs) |
|
if unversioned_files() then |
|
pkg_pfx_python2 = "py3[0-9]-" -- e.g. py39- |
|
pkg_sfx_python2 = "-py3[0-9]" |
|
pkg_pfx_python3 = "py3[0-9][0-9]-" -- e.g. py311- |
|
pkg_sfx_python3 = "-py3[0-9][0-9]" |
|
pkg_pfx_lua = "lua[0-9][0-9]-" |
|
pkg_pfx_ruby = "ruby[0-9][0-9]-" |
|
else |
|
pkg_pfx_python2 = "${PYTHON_PKGNAMEPREFIX}" |
|
pkg_sfx_python2 = "${PYTHON_PKGNAMESUFFIX}" |
|
pkg_pfx_python3 = nil |
|
pkg_sfx_python3 = nil |
|
pkg_pfx_lua = "${LUA_PKGNAMEPREFIX}" |
|
pkg_pfx_ruby = "${RUBY_PKGNAMEPREFIX}" |
|
end |
|
pkg_wildcards("^php%d%d%-", pkg_pfx_php) |
|
pkg_wildcards("-php%d%d$", pkg_sfx_php) |
|
pkg_wildcards("^phpunit%d%-", "phpunit[0-9]-") |
|
pkg_wildcards("^py3%d%-", pkg_pfx_python2, pkg_pfx_python3) |
|
pkg_wildcards("-py3%d%$", pkg_sfx_python2, pkg_sfx_python3) |
|
pkg_wildcards("^py3%d%d%-", pkg_pfx_python2, pkg_pfx_python3) |
|
pkg_wildcards("-py3%d%d%$", pkg_sfx_python2, pkg_sfx_python3) |
|
pkg_wildcards("^lua%d%d%-", pkg_pfx_lua) |
|
pkg_wildcards("-emacs_[%a_]*", "-emacs_*") |
|
pkg_wildcards("^ghostscript%d%-", "ghostscript[0-9]-") |
|
pkg_wildcards("^bacula%d%-", "bacula[0-9]-") |
|
pkg_wildcards("^bacula%d%d%-", "bacula[0-9][0-9]-") |
|
pkg_wildcards("^bareos%d%-", "bareos[0-9]-") |
|
pkg_wildcards("^bareos%d%d%-", "bareos[0-9][0-9]-") |
|
pkg_wildcards("^moosefs%d%-", "moosefs[0-9]-") |
|
pkg_wildcards("^ruby%d+-", pkg_pfx_ruby) |
|
return table_sort_uniq(pkgs) |
|
end |
|
|
|
------------------------------------------------------------------- |
|
for _, port in ipairs(PORT_LIST) do |
|
local function merge_table(t1, t2) |
|
table.move(t2, 1, #t2, #t1 + 1, t1) |
|
end |
|
local port_conflicts = {} |
|
local files = {} |
|
local msg_files = {} |
|
local conflict_pkgs = {} |
|
local function merge_data(origin) |
|
local pkgbase = PKGBASE[origin] |
|
if not BAD_FILE_PKGS[pkgbase] then |
|
local pkg_confl_file1, pkg_confl_files = conflicting_files(pkgbase, CONFLICT_PKGS[pkgbase]) |
|
merge_table(msg_files, pkg_confl_file1) -- 1 file per flavor |
|
merge_table(files, pkg_confl_files) -- all conflicting files |
|
merge_table(conflict_pkgs, CONFLICT_PKGS[pkgbase]) |
|
merge_table(port_conflicts, fetch_port_conflicts(origin)) |
|
end |
|
end |
|
local flavors = FLAVORS[port] |
|
if flavors then |
|
for _, flavor in ipairs(flavors) do |
|
merge_data(port .. "@" .. flavor) |
|
end |
|
else |
|
merge_data(port) |
|
end |
|
files = table_sort_uniq(files) |
|
msg_files = table_sort_uniq(msg_files) |
|
conflict_pkgs = generalize_patterns(conflict_pkgs, files) |
|
if #port_conflicts then |
|
port_conflicts = table_sort_uniq(port_conflicts) |
|
conflict_pkgs = conflicts_delta(port_conflicts, conflict_pkgs) |
|
end |
|
if conflict_pkgs then |
|
local conflicts_string_cur = table.concat(port_conflicts, " ") |
|
local conflicts_string_new = table.concat(conflict_pkgs, " ") |
|
local file_list = table.concat(msg_files, " ") |
|
print("# Port: " .. port) |
|
print("# Files: " .. file_list) |
|
if conflicts_string_cur ~= "" then |
|
print("# < " .. conflicts_string_cur) |
|
end |
|
print("# > " .. conflicts_string_new) |
|
print("portedit merge -ie 'CONFLICTS_INSTALL=" .. conflicts_string_new .. |
|
" # " .. file_list .. "' /usr/ports/" .. port) |
|
print() |
|
end |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local BAD_FILES_ORIGINS = {} |
|
|
|
for _, origin in ipairs(ORIGIN_LIST) do |
|
local pkgbase = PKGBASE[origin] |
|
local files = BAD_FILE_PKGS[pkgbase] |
|
if files then |
|
for _, file in ipairs(files) do |
|
if DUPLICATE_FILE[file] then |
|
local port = string.match(origin, "([^@]+)@?") |
|
BAD_FILES_ORIGINS[port] = BAD_FILES_ORIGINS[origin] or {} |
|
table.insert(BAD_FILES_ORIGINS[port], file) |
|
end |
|
end |
|
end |
|
end |
|
|
|
------------------------------------------------------------------- |
|
local bad_origins = table_sorted_keys(BAD_FILES_ORIGINS) |
|
|
|
if #bad_origins > 0 then |
|
print ("# Ports with badly named files:") |
|
print () |
|
for _, port in ipairs(bad_origins) do |
|
print ("# " .. port) |
|
local files = BAD_FILES_ORIGINS[port] |
|
table.sort(files) |
|
for _, file in ipairs(files) do |
|
print ("#", file) |
|
end |
|
print() |
|
end |
|
end
|
|
|