diff --git a/tools/pkgbase/README.md b/tools/pkgbase/README.md new file mode 100644 index 000000000000..98bb37ed32e1 --- /dev/null +++ b/tools/pkgbase/README.md @@ -0,0 +1,61 @@ +`metalog.lua` is a script that reads METALOG file created by pkgbase +(make packages) and generates reports about the installed system +and issues + +the script accepts an mtree file in a format that's returned by +`mtree -c | mtree -C` + +synopsis: +``` +metalog_reader.lua [-h] [-a | -c | -p [-count] [-size] [-f...]] [-W...] [-v] metalog-path +``` + +options: + +* `-a` prints all scan results. this is the default option if no option is + provided. +* `-c` lints the file and gives warnings/errors, including duplication and + conflicting metadata + * `-Wcheck-notagdir` entries with dir type and no tags will be also included + the first time they appear (1) +* `-p` list all package names found in the file as exactly specified by + `tags=package=...` + * `-count` display the number of files of the package + * `-size` display the size of the package + * `-fsetgid` only include packages with setgid files + * `-fsetuid` only include packages with setuid files + * `-fsetid` only include packages with setgid or setuid files +* `-v` verbose mode +* `-h` help page + +some examples: + +* `metalog_reader.lua -a METALOG` + prints all scan results described below. this is the default option +* `metalog_reader.lua -c METALOG` + only prints errors and warnings found in the file +* `metalog_reader.lua -c -Wcheck-notagdir METALOG` + prints errors and warnings found in the file, including directories with no + tags +* `metalog_reader.lua -p METALOG` + only prints all the package names found in the file +* `metalog_reader.lua -p -count -size METALOG` + prints all the package names, followed by number of files, followed by total + size +* `metalog_reader.lua -p -size -fsetid METALOG` + prints packages that has either setuid/setgid files, followed by the total + size +* `metalog_reader.lua -p -fsetuid -fsetgid METALOG` + prints packages that has both setuid and setgid files (if more than one + filters are specified, they are composed using logic and) +* `metalog_reader.lua -p -count -size -fsetuid METALOG` + prints packages that has setuid files, followed by number of files and total + size + +(1) if we have two entries +``` +./bin type=dir uname=root gname=wheel mode=0755 +./bin type=dir uname=root gname=wheel mode=0755 tags=... +``` +by default, this is not warned. if the option is enabled, this will be warned +as the second line sufficiently covers the first line. diff --git a/tools/pkgbase/metalog_reader.lua b/tools/pkgbase/metalog_reader.lua new file mode 100644 index 000000000000..a429c981a71d --- /dev/null +++ b/tools/pkgbase/metalog_reader.lua @@ -0,0 +1,521 @@ +#!/usr/libexec/flua + +-- SPDX-License-Identifier: BSD-2-Clause-FreeBSD +-- +-- Copyright(c) 2020 The FreeBSD Foundation. +-- +-- Redistribution and use in source and binary forms, with or without +-- modification, are permitted provided that the following conditions +-- are met: +-- 1. Redistributions of source code must retain the above copyright +-- notice, this list of conditions and the following disclaimer. +-- 2. Redistributions in binary form must reproduce the above copyright +-- notice, this list of conditions and the following disclaimer in the +-- documentation and/or other materials provided with the distribution. +-- +-- THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +-- ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +-- IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +-- ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +-- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +-- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +-- OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +-- HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +-- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +-- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +-- SUCH DAMAGE. + +-- $FreeBSD$ + +function main(args) + if #args == 0 then usage() end + local filename + local printall, checkonly, pkgonly = + #args == 1, false, false + local dcount, dsize, fuid, fgid, fid = + false, false, false, false, false + local verbose = false + local w_notagdirs = false + + local i = 1 + while i <= #args do + if args[i] == '-h' then + usage(true) + elseif args[i] == '-a' then + printall = true + elseif args[i] == '-c' then + printall = false + checkonly = true + elseif args[i] == '-p' then + printall = false + pkgonly = true + while i < #args do + i = i+1 + if args[i] == '-count' then + dcount = true + elseif args[i] == '-size' then + dsize = true + elseif args[i] == '-fsetuid' then + fuid = true + elseif args[i] == '-fsetgid' then + fgid = true + elseif args[i] == '-fsetid' then + fid = true + else + i = i-1 + break + end + end + elseif args[i] == '-v' then + verbose = true + elseif args[i] == '-Wcheck-notagdir' then + w_notagdirs = true + elseif args[i]:match('^%-') then + io.stderr:write('Unknown argument '..args[i]..'.\n') + usage() + else + filename = args[i] + end + i = i+1 + end + + if filename == nil then + io.stderr:write('Missing filename.\n') + usage() + end + + local sess = Analysis_session(filename, verbose, w_notagdirs) + + if printall then + io.write('--- PACKAGE REPORTS ---\n') + io.write(sess.pkg_report_full()) + io.write('--- LINTING REPORTS ---\n') + print_lints(sess) + elseif checkonly then + print_lints(sess) + elseif pkgonly then + io.write(sess.pkg_report_simple(dcount, dsize, { + fuid and sess.pkg_issetuid or nil, + fgid and sess.pkg_issetgid or nil, + fid and sess.pkg_issetid or nil + })) + else + io.stderr:write('This text should not be displayed.') + usage() + end +end + +--- @param man boolean +function usage(man) + local sn = 'Usage: '..arg[0].. ' [-h] [-a | -c | -p [-count] [-size] [-f...]] [-W...] metalog-path \n' + if man then + io.write('\n') + io.write(sn) + io.write( +[[ + +The script reads METALOG file created by pkgbase (make packages) and generates +reports about the installed system and issues. It accepts an mtree file in a +format that's returned by `mtree -c | mtree -C` + + Options: + -a prints all scan results. this is the default option if no option + is provided. + -c lints the file and gives warnings/errors, including duplication + and conflicting metadata + -Wcheck-notagdir entries with dir type and no tags will be also + included the first time they appear + -p list all package names found in the file as exactly specified by + `tags=package=...` + -count display the number of files of the package + -size display the size of the package + -fsetgid only include packages with setgid files + -fsetuid only include packages with setuid files + -fsetid only include packages with setgid or setuid files + -v verbose mode + -h help page + +]]) + os.exit() + else + io.stderr:write(sn) + os.exit(1) + end +end + +--- @param sess Analysis_session +function print_lints(sess) + local dupwarn, duperr = sess.dup_report() + io.write(dupwarn) + io.write(duperr) + local inodewarn, inodeerr = sess.inode_report() + io.write(inodewarn) + io.write(inodeerr) +end + +--- @param t table +function sortedPairs(t) + local sortedk = {} + for k in next, t do sortedk[#sortedk+1] = k end + table.sort(sortedk) + local i = 0 + return function() + i = i + 1 + return sortedk[i], t[sortedk[i]] + end +end + +--- @param t table +--- @param f function U> +function table_map(t, f) + local res = {} + for k, v in pairs(t) do res[k] = f(v) end + return res +end + +--- @class MetalogRow +-- a table contaning file's info, from a line content from METALOG file +-- all fields in the table are strings +-- sample output: +-- { +-- filename = ./usr/share/man/man3/inet6_rthdr_segments.3.gz +-- lineno = 5 +-- attrs = { +-- gname = 'wheel' +-- uname = 'root' +-- mode = '0444' +-- size = '1166' +-- time = nil +-- type = 'file' +-- tags = 'package=clibs,debug' +-- } +-- } +--- @param line string +function MetalogRow(line, lineno) + local res, attrs = {}, {} + local filename, rest = line:match('^(%S+) (.+)$') + -- mtree file has space escaped as '\\040', not affecting splitting + -- string by space + for attrpair in rest:gmatch('[^ ]+') do + local k, v = attrpair:match('^(.-)=(.+)') + attrs[k] = v + end + res.filename = filename + res.linenum = lineno + res.attrs = attrs + return res +end + +-- check if an array of MetalogRows are equivalent. if not, the first field +-- that's different is returned secondly +--- @param rows MetalogRow[] +--- @param ignore_name boolean +--- @param ignore_tags boolean +function metalogrows_all_equal(rows, ignore_name, ignore_tags) + local __eq = function(l, o) + if not ignore_name and l.filename ~= o.filename then + return false, 'filename' + end + -- ignoring linenum in METALOG file as it's not relavant + for k in pairs(l.attrs) do + if ignore_tags and k == 'tags' then goto continue end + if l.attrs[k] ~= o.attrs[k] and o.attrs[k] ~= nil then + return false, k + end + ::continue:: + end + return true + end + for _, v in ipairs(rows) do + local bol, offby = __eq(v, rows[1]) + if not bol then return false, offby end + end + return true +end + +--- @param tagstr string +function pkgname_from_tag(tagstr) + local ext, pkgname, pkgend = '', '', '' + for seg in tagstr:gmatch('[^,]+') do + if seg:match('package=') then + pkgname = seg:sub(9) + elseif seg == 'development' or seg == 'profile' + or seg == 'debug' or seg == 'docs' then + pkgend = seg + else + ext = ext == '' and seg or ext..'-'..seg + end + end + pkgname = pkgname + ..(ext == '' and '' or '-'..ext) + ..(pkgend == '' and '' or '-'..pkgend) + return pkgname +end + +--- @class Analysis_session +--- @param metalog string +--- @param verbose boolean +--- @param w_notagdirs boolean turn on to also check directories +function Analysis_session(metalog, verbose, w_notagdirs) + local files = {} -- map + -- set is map. if bool is true then elem exists + local pkgs = {} -- map> + ----- used to keep track of files not belonging to a pkg. not used so + ----- it is commented with ----- + -----local nopkg = {} -- set + --- @public + local swarn = {} + --- @public + local serrs = {} + + -- returns number of files in package and size of package + -- nil is returned upon errors + --- @param pkgname string + local function pkg_size(pkgname) + local filecount, sz = 0, 0 + for filename in pairs(pkgs[pkgname]) do + local rows = files[filename] + -- normally, there should be only one row per filename + -- if these rows are equal, there should be warning, but it + -- does not affect size counting. if not, it is an error + if #rows > 1 and not metalogrows_all_equal(rows) then + return nil + end + local row = rows[1] + if row.attrs.type == 'file' then + sz = sz + tonumber(row.attrs.size) + end + filecount = filecount + 1 + end + return filecount, sz + end + + --- @param pkgname string + --- @param mode number + local function pkg_ismode(pkgname, mode) + for filename in pairs(pkgs[pkgname]) do + for _, row in ipairs(files[filename]) do + if tonumber(row.attrs.mode, 8) & mode ~= 0 then + return true + end + end + end + return false + end + + --- @param pkgname string + --- @public + local function pkg_issetuid(pkgname) + return pkg_ismode(pkgname, 2048) + end + + --- @param pkgname string + --- @public + local function pkg_issetgid(pkgname) + return pkg_ismode(pkgname, 1024) + end + + --- @param pkgname string + --- @public + local function pkg_issetid(pkgname) + return pkg_issetuid(pkgname) or pkg_issetgid(pkgname) + end + + -- sample return: + -- { [*string]: { count=1, size=2, issetuid=true, issetgid=true } } + local function pkg_report_helper_table() + local res = {} + for pkgname in pairs(pkgs) do + res[pkgname] = {} + res[pkgname].count, + res[pkgname].size = pkg_size(pkgname) + res[pkgname].issetuid = pkg_issetuid(pkgname) + res[pkgname].issetgid = pkg_issetgid(pkgname) + end + return res + end + + -- returns a string describing package scan report + --- @public + local function pkg_report_full() + local sb = {} + for pkgname, v in sortedPairs(pkg_report_helper_table()) do + sb[#sb+1] = 'Package '..pkgname..':' + if v.issetuid or v.issetgid then + sb[#sb+1] = ''..table.concat({ + v.issetuid and ' setuid' or '', + v.issetgid and ' setgid' or '' }, '') + end + sb[#sb+1] = '\n number of files: '..(v.count or '?') + ..'\n total size: '..(v.size or '?') + sb[#sb+1] = '\n' + end + return table.concat(sb, '') + end + + --- @param have_count boolean + --- @param have_size boolean + --- @param filters function[] + --- @public + -- returns a string describing package size report. + -- sample: "mypackage 2 2048"* if both booleans are true + local function pkg_report_simple(have_count, have_size, filters) + filters = filters or {} + local sb = {} + for pkgname, v in sortedPairs(pkg_report_helper_table()) do + local pred = true + -- doing a foldl to all the function results with (and) + for _, f in pairs(filters) do pred = pred and f(pkgname) end + if pred then + sb[#sb+1] = pkgname..table.concat({ + have_count and (' '..(v.count or '?')) or '', + have_size and (' '..(v.size or '?')) or ''}, '') + ..'\n' + end + end + return table.concat(sb, '') + end + + -- returns a string describing duplicate file warnings, + -- returns a string describing duplicate file errors + --- @public + local function dup_report() + local warn, errs = {}, {} + for filename, rows in sortedPairs(files) do + if #rows == 1 then goto continue end + local iseq, offby = metalogrows_all_equal(rows) + if iseq then -- repeated line, just a warning + warn[#warn+1] = 'warning: '..filename + ..' repeated with same meta: line ' + ..table.concat( + table_map(rows, function(e) return e.linenum end), ',') + warn[#warn+1] = '\n' + elseif not metalogrows_all_equal(rows, false, true) then + -- same filename (possibly different tags), different metadata, an error + errs[#errs+1] = 'error: '..filename + ..' exists in multiple locations and with different meta: line ' + ..table.concat( + table_map(rows, function(e) return e.linenum end), ',') + ..'. off by "'..offby..'"' + errs[#errs+1] = '\n' + end + ::continue:: + end + return table.concat(warn, ''), table.concat(errs, '') + end + + -- returns a string describing warnings of found hard links + -- returns a string describing errors of found hard links + --- @public + local function inode_report() + -- obtain inodes of filenames + local attributes = require('lfs').attributes + local inm = {} -- map + local unstatables = {} -- string[] + for filename in pairs(files) do + -- i only took the first row of a filename, + -- and skip links and folders + if files[filename][1].attrs.type ~= 'file' then + goto continue + end + -- make ./xxx become /xxx so that we can stat + filename = filename:sub(2) + local fs = attributes(filename) + if fs == nil then + unstatables[#unstatables+1] = filename + goto continue + end + local inode = fs.ino + inm[inode] = inm[inode] or {} + -- add back the dot prefix + table.insert(inm[inode], '.'..filename) + ::continue:: + end + + local warn, errs = {}, {} + for _, filenames in pairs(inm) do + if #filenames == 1 then goto continue end + -- i only took the first row of a filename + local rows = table_map(filenames, function(e) + return files[e][1] + end) + local iseq, offby = metalogrows_all_equal(rows, true, true) + if not iseq then + errs[#errs+1] = 'error: ' + ..'entries point to the same inode but have different meta: ' + ..table.concat(filenames, ',')..' in line ' + ..table.concat( + table_map(rows, function(e) return e.linenum end), ',') + ..'. off by "'..offby..'"' + errs[#errs+1] = '\n' + end + ::continue:: + end + + if #unstatables > 0 then + warn[#warn+1] = verbose and + 'note: skipped checking inodes: '..table.concat(unstatables, ',')..'\n' + or + 'note: skipped checking inodes for '..#unstatables..' entries\n' + end + + return table.concat(warn, ''), table.concat(errs, '') + end + + do + local fp, errmsg, errcode = io.open(metalog, 'r') + if fp == nil then + io.stderr:write('cannot open '..metalog..': '..errmsg..': '..errcode..'\n') + os.exit(1) + end + + -- scan all lines and put file data into the dictionaries + local firsttimes = {} -- set + local lineno = 0 + for line in fp:lines() do + -----local isinpkg = false + lineno = lineno + 1 + -- skip lines begining with # + if line:match('^%s*#') then goto continue end + -- skip blank lines + if line:match('^%s*$') then goto continue end + + local data = MetalogRow(line, lineno) + -- entries with dir and no tags... ignore for the first time + if not w_notagdirs and + data.attrs.tags == nil and data.attrs.type == 'dir' + and not firsttimes[data.filename] then + firsttimes[data.filename] = true + goto continue + end + + files[data.filename] = files[data.filename] or {} + table.insert(files[data.filename], data) + + if data.attrs.tags ~= nil then + pkgname = pkgname_from_tag(data.attrs.tags) + pkgs[pkgname] = pkgs[pkgname] or {} + pkgs[pkgname][data.filename] = true + ------isinpkg = true + end + -----if not isinpkg then nopkg[data.filename] = true end + ::continue:: + end + + fp:close() + end + + return { + warn = swarn, + errs = serrs, + pkg_issetuid = pkg_issetuid, + pkg_issetgid = pkg_issetgid, + pkg_issetid = pkg_issetid, + pkg_report_full = pkg_report_full, + pkg_report_simple = pkg_report_simple, + dup_report = dup_report, + inode_report = inode_report + } +end + +main(arg)