From 53690d83be1ddb56c145c4e678e44632167fba71 Mon Sep 17 00:00:00 2001
From: Matthias Schiffer <mschiffer@universe-factory.net>
Date: Sun, 3 May 2020 21:35:01 +0200
Subject: [PATCH] build: move package list generation to target_config_lib.lua,
fix precedence
The precedence of different package lists was broken since #1876,
disallowing removal of GLUON_FEATURES packages via GLUON_SITE_PACKAGES.
Including all package selections, both implicit defaults and explicit
handling in Gluon, the order of precedence is now the following:
1. OpenWrt defaults (including target-specific defaults)
2. Device-specific packages from OpenWrt
3. Generic default packages (from target/generic)
4. Target default packages (target/$(GLUON_TARGET))
5. Removal of opkg for tiny targets
6. Packages derived from GLUON_FEATURES + GLUON_FEATURES_$(class)
7. GLUON_SITE_PACKAGES
8. GLUON_SITE_PACKAGES_$(class)
9. Device-specific packages from target/$(GLUON_TARGET)
10. Device-specific packages from GLUON_$(device)_SITE_PACKAGES
This also contains various pieces of cleanup:
- No hardcoded order of device classes for target_config.lua arguments
anymore (in fact, the Makefile doesn't know anything about device
classes now)
- target_conifg_lib.lua only hardcodes the fallback class for x86, no
other occurences of specific class names
- Feature -> package list mapping is moved from Makefile to the Lua code
as well (still implemented in Shell though)
---
Makefile | 42 ++-------
scripts/copy_output.lua | 5 +-
scripts/target_config_lib.lua | 161 ++++++++++++++++++++++++----------
scripts/target_lib.lua | 7 +-
targets/x86-geode | 2 -
targets/x86.inc | 2 -
6 files changed, 124 insertions(+), 95 deletions(-)
diff --git a/Makefile b/Makefile
index ab52cc9ac..f614f3d9f 100644
--- a/Makefile
+++ b/Makefile
@@ -55,7 +55,8 @@ endef
GLUON_VARS = \
GLUON_RELEASE GLUON_REGION GLUON_MULTIDOMAIN GLUON_DEBUG GLUON_DEPRECATED GLUON_DEVICES \
GLUON_TARGETSDIR GLUON_PATCHESDIR GLUON_TMPDIR GLUON_IMAGEDIR GLUON_PACKAGEDIR \
- GLUON_SITEDIR GLUON_RELEASE GLUON_BRANCH GLUON_LANGS GLUON_BASE_FEEDS BOARD SUBTARGET
+ GLUON_SITEDIR GLUON_RELEASE GLUON_BRANCH GLUON_LANGS GLUON_BASE_FEEDS \
+ GLUON_TARGET BOARD SUBTARGET
unexport $(GLUON_VARS)
GLUON_ENV = $(foreach var,$(GLUON_VARS),$(var)=$(call escape,$($(var))))
@@ -135,31 +136,6 @@ lint-lua: FORCE
lint-sh: FORCE
scripts/lint-sh.sh
-define merge_lists
- $(1) :=
- $(foreach pkg,$(2),
- $(1) := $$(strip $$(filter-out -$$(patsubst -%,%,$(pkg)) $$(patsubst -%,%,$(pkg)),$$(value $(1))) $(pkg))
- )
-endef
-
-define feature_packages
- $(1) := $(shell scripts/features.sh '$(2)' || echo '__ERROR__')
-endef
-
-$(eval $(call merge_lists,GLUON_FEATURE_LIST_standard,$(GLUON_FEATURES) $(GLUON_FEATURES_standard)))
-$(eval $(call merge_lists,GLUON_FEATURE_LIST_tiny,$(GLUON_FEATURES) $(GLUON_FEATURES_tiny)))
-
-$(eval $(call feature_packages,GLUON_FEATURE_PACKAGES_standard,$(GLUON_FEATURE_LIST_standard)))
-$(eval $(call feature_packages,GLUON_FEATURE_PACKAGES_tiny,$(GLUON_FEATURE_LIST_tiny)))
-
-ifneq ($(filter __ERROR__,$(GLUON_FEATURES_standard) $(GLUON_FEATURES_tiny)),)
- $(error Error while evaluating features)
-endif
-
-$(eval $(call merge_lists,GLUON_DEFAULT_PACKAGES,$(GLUON_DEFAULT_PACKAGES) $(GLUON_SITE_PACKAGES)))
-$(eval $(call merge_lists,GLUON_CLASS_PACKAGES_standard,$(GLUON_FEATURE_PACKAGES_standard) $(GLUON_SITE_PACKAGES_standard)))
-$(eval $(call merge_lists,GLUON_CLASS_PACKAGES_tiny,$(GLUON_FEATURE_PACKAGES_tiny) $(GLUON_SITE_PACKAGES_tiny)))
-
LUA := openwrt/staging_dir/hostpkg/bin/lua
@@ -182,22 +158,16 @@ config: $(LUA) FORCE
$(call CheckSite,$(conf)); \
)
- $(GLUON_ENV) \
- $(LUA) scripts/target_config.lua '$(GLUON_TARGET)' '$(GLUON_DEFAULT_PACKAGES)' '$(GLUON_CLASS_PACKAGES_standard)' '$(GLUON_CLASS_PACKAGES_tiny)' \
- > openwrt/.config
+ $(GLUON_ENV) $(LUA) scripts/target_config.lua > openwrt/.config
$(OPENWRTMAKE) defconfig
-
- $(GLUON_ENV) \
- $(LUA) scripts/target_config_check.lua '$(GLUON_TARGET)' '$(GLUON_DEFAULT_PACKAGES)' '$(GLUON_CLASS_PACKAGES_standard)' '$(GLUON_CLASS_PACKAGES_tiny)'
+ $(GLUON_ENV) $(LUA) scripts/target_config_check.lua
all: config
+
- $(GLUON_ENV) \
- $(LUA) scripts/clean_output.lua
+ $(GLUON_ENV) $(LUA) scripts/clean_output.lua
$(OPENWRTMAKE)
- $(GLUON_ENV) \
- $(LUA) scripts/copy_output.lua '$(GLUON_TARGET)'
+ $(GLUON_ENV) $(LUA) scripts/copy_output.lua
clean download: config
+$(OPENWRTMAKE) $@
diff --git a/scripts/copy_output.lua b/scripts/copy_output.lua
index 78300aa79..91547964f 100755
--- a/scripts/copy_output.lua
+++ b/scripts/copy_output.lua
@@ -1,12 +1,13 @@
local lib = dofile('scripts/target_lib.lua')
local env = lib.env
+local target = env.GLUON_TARGET
+
+assert(target)
assert(env.GLUON_IMAGEDIR)
assert(env.GLUON_PACKAGEDIR)
-local target = arg[1]
-
local openwrt_target
local subtarget = env.SUBTARGET
if subtarget ~= '' then
diff --git a/scripts/target_config_lib.lua b/scripts/target_config_lib.lua
index c52b7e309..ee2bf0069 100644
--- a/scripts/target_config_lib.lua
+++ b/scripts/target_config_lib.lua
@@ -1,17 +1,55 @@
+-- Split a string into words
+local function split(s)
+ local ret = {}
+ for w in string.gmatch(s, '%S+') do
+ table.insert(ret, w)
+ end
+ return ret
+end
+
+-- Strip leading '-' character
+local function strip_neg(s)
+ if string.sub(s, 1, 1) == '-' then
+ return string.sub(s, 2)
+ else
+ return s
+ end
+end
+
+-- Add an element to a list, removing duplicate entries and handling negative
+-- elements prefixed with a '-'
+local function append_to_list(list, item, keep_neg)
+ local match = strip_neg(item)
+ local ret = {}
+ for _, el in ipairs(list) do
+ if strip_neg(el) ~= match then
+ table.insert(ret, el)
+ end
+ end
+ if keep_neg ~= false or string.sub(item, 1, 1) ~= '-' then
+ table.insert(ret, item)
+ end
+ return ret
+end
+
+local function compact_list(list, keep_neg)
+ local ret = {}
+ for _, el in ipairs(list) do
+ ret = append_to_list(ret, el, keep_neg)
+ end
+ return ret
+end
+
return function(funcs)
local lib = dofile('scripts/target_lib.lua')
local env = lib.env
+ local target = env.GLUON_TARGET
+
+ assert(target)
assert(env.BOARD)
assert(env.SUBTARGET)
- local target = arg[1]
- local extra_packages = arg[2]
- local class_packages = {
- standard = arg[3],
- tiny = arg[4],
- }
-
local openwrt_config_target
if env.SUBTARGET ~= '' then
openwrt_config_target = env.BOARD .. '_' .. env.SUBTARGET
@@ -32,73 +70,102 @@ END_MAKE
end
local function site_packages(image)
- return site_vars(string.format('$(GLUON_%s_SITE_PACKAGES)', image))
+ return split(site_vars(string.format('$(GLUON_%s_SITE_PACKAGES)', image)))
+ end
+
+ -- TODO: Rewrite features.sh in Lua
+ local function feature_packages(features)
+ -- Ugly hack: Lua doesn't give us the return code of a popened
+ -- command, so we match on a special __ERROR__ marker
+ local pkgs = lib.exec_capture({'scripts/features.sh', features}, '|| echo __ERROR__')
+ assert(string.find(pkgs, '__ERROR__') == nil, 'Error while evaluating features')
+ return pkgs
+ end
+
+ -- This involves running lots of processes to evaluate site.mk, so we
+ -- add a simple cache
+ local class_cache = {}
+ local function class_packages(class)
+ if class_cache[class] then
+ return class_cache[class]
+ end
+
+ local features = site_vars(string.format('$(GLUON_FEATURES) $(GLUON_FEATURES_%s)', class))
+ features = table.concat(compact_list(split(features), false), ' ')
+
+ local pkgs = feature_packages(features)
+ pkgs = pkgs .. ' ' .. site_vars(string.format('$(GLUON_SITE_PACKAGES) $(GLUON_SITE_PACKAGES_%s)', class))
+
+ pkgs = compact_list(split(pkgs))
+
+ class_cache[class] = pkgs
+ return pkgs
end
local function handle_target_pkgs(pkgs)
- local packages = string.gmatch(pkgs, '%S+')
- for pkg in packages do
- lib.packages {pkg}
+ for _, pkg in ipairs(pkgs) do
+ if string.sub(pkg, 1, 1) == '-' then
+ lib.try_config('# CONFIG_PACKAGE_%s is not set', string.sub(pkg, 2))
+ else
+ funcs.config_package(lib.config, pkg, 'y')
+ end
end
end
lib.include('generic')
- handle_target_pkgs(extra_packages)
lib.include(target)
- if lib.target_class ~= nil then
- handle_target_pkgs(class_packages[lib.target_class])
- end
-
lib.check_devices()
-
if not lib.opkg then
lib.config '# CONFIG_SIGNED_PACKAGES is not set'
lib.config 'CONFIG_CLEAN_IPKG=y'
lib.packages {'-opkg'}
end
+ if #lib.devices > 0 then
+ handle_target_pkgs(lib.target_packages)
- local default_pkgs = ''
- for _, pkg in ipairs(lib.target_packages) do
- default_pkgs = default_pkgs .. ' ' .. pkg
+ for _, dev in ipairs(lib.devices) do
+ local profile = dev.options.profile or dev.name
- if string.sub(pkg, 1, 1) == '-' then
- lib.try_config('# CONFIG_PACKAGE_%s is not set', string.sub(pkg, 2))
- else
- funcs.config_package(lib.config, pkg, 'y')
- end
- end
+ local device_pkgs = {}
+ local function handle_pkgs(pkgs)
+ for _, pkg in ipairs(pkgs) do
+ if string.sub(pkg, 1, 1) ~= '-' then
+ funcs.config_package(lib.config, pkg, 'm')
+ end
+ device_pkgs = append_to_list(device_pkgs, pkg)
+ end
+ end
- for _, dev in ipairs(lib.devices) do
- local profile = dev.options.profile or dev.name
- local device_pkgs = default_pkgs
+ handle_pkgs(lib.target_packages)
+ handle_pkgs(class_packages(dev.options.class))
+ handle_pkgs(dev.options.packages or {})
+ handle_pkgs(site_packages(dev.image))
- local function handle_pkg(pkg)
- if string.sub(pkg, 1, 1) ~= '-' then
- funcs.config_package(lib.config, pkg, 'm')
- end
- device_pkgs = device_pkgs .. ' ' .. pkg
+ funcs.config_message(lib.config, string.format("unable to enable device '%s'", profile),
+ 'CONFIG_TARGET_DEVICE_%s_DEVICE_%s=y', openwrt_config_target, profile)
+ lib.config('CONFIG_TARGET_DEVICE_PACKAGES_%s_DEVICE_%s="%s"',
+ openwrt_config_target, profile,
+ table.concat(device_pkgs, ' '))
end
+ else
+ -- x86 fallback: no devices
+ local target_pkgs = {}
local function handle_pkgs(pkgs)
- local packages = string.gmatch(pkgs or '', '%S+')
- for pkg in packages do
- handle_pkg(pkg)
+ for _, pkg in ipairs(pkgs) do
+ target_pkgs = append_to_list(target_pkgs, pkg)
end
end
- for _, pkg in ipairs(dev.options.packages or {}) do
- handle_pkg(pkg)
- end
- handle_pkgs(site_packages(dev.image))
-
- handle_pkgs(class_packages[dev.options.class])
+ -- Just hardcode the class for device-less targets to 'standard'
+ -- - this is x86 only at the moment, and it will have devices
+ -- in OpenWrt 19.07 + 1 as well
+ handle_pkgs(lib.target_packages)
+ handle_pkgs(class_packages('standard'))
- funcs.config_message(lib.config, string.format("unable to enable device '%s'", profile),
- 'CONFIG_TARGET_DEVICE_%s_DEVICE_%s=y', openwrt_config_target, profile)
- lib.config('CONFIG_TARGET_DEVICE_PACKAGES_%s_DEVICE_%s="%s"',
- openwrt_config_target, profile, device_pkgs)
+ handle_target_pkgs(target_pkgs)
end
return lib
diff --git a/scripts/target_lib.lua b/scripts/target_lib.lua
index d669928f7..0fc2d8b61 100644
--- a/scripts/target_lib.lua
+++ b/scripts/target_lib.lua
@@ -23,7 +23,6 @@ assert(env.GLUON_DEPRECATED)
M.site_code = assert(assert(dofile('scripts/site_config.lua')('site.conf')).site_code)
M.target_packages = {}
-M.target_class = nil
M.configs = {}
M.devices = {}
M.images = {}
@@ -40,7 +39,7 @@ local default_options = {
aliases = {},
manifest_aliases = {},
packages = {},
- class = "standard",
+ class = 'standard',
deprecated = false,
broken = false,
}
@@ -159,10 +158,6 @@ function F.config(...)
M.configs[string.format(...)] = 2
end
-function F.class(target_class)
- M.target_class = target_class
-end
-
function F.packages(pkgs)
for _, pkg in ipairs(pkgs) do
table.insert(M.target_packages, pkg)
diff --git a/targets/x86-geode b/targets/x86-geode
index 4f13f9a93..57519eb1b 100644
--- a/targets/x86-geode
+++ b/targets/x86-geode
@@ -1,5 +1,3 @@
-class 'standard'
-
packages {
'kmod-3c59x',
'kmod-8139cp',
diff --git a/targets/x86.inc b/targets/x86.inc
index 461f48395..9536e805b 100644
--- a/targets/x86.inc
+++ b/targets/x86.inc
@@ -1,8 +1,6 @@
config 'CONFIG_VDI_IMAGES=y'
config 'CONFIG_VMDK_IMAGES=y'
-class 'standard'
-
packages {
'kmod-3c59x',
'kmod-8139cp',
--
GitLab