Nabble - debian-dak

[dak/master] fix up cross-component logic

2009-11-28T15:57:25Z

Signed-off-by: Mark Hymers <mhy@...>
---
daklib/dbconn.py | 2 +-
daklib/queue.py | 37 +++++++++++++++++++++++++++++++------
2 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/daklib/dbconn.py b/daklib/dbconn.py
index 1418d08..2b2d01d 100644
--- a/daklib/dbconn.py
+++ b/daklib/dbconn.py
@@ -2291,7 +2291,7 @@ def add_dsc_to_db(u, filename, session=None):

session.flush()

- return dsc_component, dsc_location_id, pfs
+ return source, dsc_component, dsc_location_id, pfs

__all__.append('add_dsc_to_db')

diff --git a/daklib/queue.py b/daklib/queue.py
index d5057a6..9a238bb 100755
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -1830,7 +1830,7 @@ distribution."""
# Add the .dsc file to the DB first
for newfile, entry in self.pkg.files.items():
if entry["type"] == "dsc":
- dsc_component, dsc_location_id, pfs = add_dsc_to_db(self, newfile, session)
+ source, dsc_component, dsc_location_id, pfs = add_dsc_to_db(self, newfile, session)
for j in pfs:
poolfiles.append(j)

@@ -1842,6 +1842,7 @@ distribution."""
# If this is a sourceful diff only upload that is moving
# cross-component we need to copy the .orig files into the new
# component too for the same reasons as above.
+ # XXX: mhy: I think this should be in add_dsc_to_db
if self.pkg.changes["architecture"].has_key("source"):
for orig_file in self.pkg.orig_files.keys():
if not self.pkg.orig_files[orig_file].has_key("id"):
@@ -1861,19 +1862,43 @@ distribution."""
# TODO: Care about size/md5sum collisions etc
(found, newf) = check_poolfile(new_filename, old_dat['size'], old_dat['md5sum'], dsc_location_id, session)

+ # TODO: Uhm, what happens if newf isn't None - something has gone badly and we should cope
if newf is None:
utils.copy(old_filename, os.path.join(cnf["Dir::Pool"], new_filename))
newf = add_poolfile(new_filename, old_dat, dsc_location_id, session)

- # TODO: Check that there's only 1 here
- source = get_sources_from_name(self.pkg.changes["source"], self.pkg.changes["version"])[0]
- dscf = get_dscfiles(source_id=source.source_id, poolfile_id=orig_file_id, session=session)[0]
- dscf.poolfile_id = newf.file_id
- session.add(dscf)
session.flush()

+ # Don't reference the old file from this changes
+ for p in poolfiles:
+ if p.file_id == oldf.file_id:
+ poolfiles.remove(p)
+
poolfiles.append(newf)

+ # Fix up the DSC references
+ toremove = []
+
+ for df in source.srcfiles:
+ if df.poolfile.file_id == oldf.file_id:
+ # Add a new DSC entry and mark the old one for deletion
+ # Don't do it in the loop so we don't change the thing we're iterating over
+ newdscf = DSCFile()
+ newdscf.source_id = source.source_id
+ newdscf.poolfile_id = newf.file_id
+ session.add(newdscf)
+
+ toremove.append(df)
+
+ for df in toremove:
+ session.delete(df)
+
+ # Flush our changes
+ session.flush()
+
+ # Make sure that our source object is up-to-date
+ session.expire(source)
+
# Install the files into the pool
for newfile, entry in self.pkg.files.items():
destination = os.path.join(cnf["Dir::Pool"], entry["pool name"], newfile)
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master] fix variable references

2009-11-28T15:22:02Z

Signed-off-by: Mark Hymers <mhy@...>
---
daklib/queue.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/daklib/queue.py b/daklib/queue.py
index 0433a96..d5057a6 100755
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -1859,7 +1859,7 @@ distribution."""
new_filename = os.path.join(utils.poolify(self.pkg.changes["source"], dsc_component), os.path.basename(old_filename))

# TODO: Care about size/md5sum collisions etc
- (found, newf) = check_poolfile(new_filename, file_size, file_md5sum, dsc_location_id, session)
+ (found, newf) = check_poolfile(new_filename, old_dat['size'], old_dat['md5sum'], dsc_location_id, session)

if newf is None:
utils.copy(old_filename, os.path.join(cnf["Dir::Pool"], new_filename))
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 2/2] do not create new db views just improve older ones

2009-11-28T14:12:56Z

Signed-off-by: Torsten Werner <twerner@...>
---
dak/cruft_report.py | 54 +++++++++++++++++++++++++++++++++---------------
dak/dakdb/update27.py | 51 ----------------------------------------------
2 files changed, 37 insertions(+), 68 deletions(-)

diff --git a/dak/cruft_report.py b/dak/cruft_report.py
index 384ad0e..ef9362f 100755
--- a/dak/cruft_report.py
+++ b/dak/cruft_report.py
@@ -273,38 +273,58 @@ def do_dubious_nbs(dubious_nbs):
################################################################################

def obsolete_source(suite_name, session):
- """returns obsolete source packages for suite_name sorted by
- install_date"""
+ """returns obsolete source packages for suite_name without binaries
+ in the same suite sorted by install_date; install_date should help
+ detecting source only (or binary throw away) uploads; duplicates in
+ the suite are skipped
+
+ subquery 'source_suite_unique' returns source package names from
+ suite without duplicates; the rationale behind is that neither
+ cruft-report nor rm cannot handle duplicates (yet)"""

query = """
-SELECT os.src, os.source, os.version, os.install_date
- FROM obsolete_source os
- JOIN suite s on s.id = os.suite
- WHERE s.suite_name = :suite_name
+WITH source_suite_unique AS
+ (SELECT source, suite
+ FROM source_suite GROUP BY source, suite HAVING count(*) = 1)
+SELECT ss.src, ss.source, ss.version,
+ to_char(ss.install_date, 'YYYY-MM-DD') AS install_date
+ FROM source_suite ss
+ JOIN source_suite_unique ssu
+ ON ss.source = ssu.source AND ss.suite = ssu.suite
+ JOIN suite s ON s.id = ss.suite
+ LEFT JOIN bin_associations_binaries bab
+ ON ss.src = bab.source AND ss.suite = bab.suite
+ WHERE s.suite_name = :suite_name AND bab.id IS NULL
ORDER BY install_date"""
args = { 'suite_name': suite_name }
return session.execute(query, args)

def source_bin(source, session):
- """returns binaries built by source for all or no suite"""
+ """returns binaries built by source for all or no suite grouped and
+ ordered by package name"""

query = """
-SELECT package
- FROM source_bin
- WHERE source = :source
- ORDER BY package"""
+SELECT b.package
+ FROM binaries b
+ JOIN src_associations_src sas ON b.source = sas.src
+ WHERE sas.source = :source
+ GROUP BY b.package
+ ORDER BY b.package"""
args = { 'source': source }
return session.execute(query, args)

def newest_source_bab(suite_name, package, session):
- """returns newest source that builds binary package in suite"""
+ """returns newest source that builds binary package in suite grouped
+ and sorted by source and package name"""

query = """
-SELECT source, srcver
- FROM newest_source_bab nsb
- JOIN suite s on s.id = nsb.suite
- WHERE s.suite_name = :suite_name AND nsb.package = :package
- ORDER BY source"""
+SELECT sas.source, MAX(sas.version) AS srcver
+ FROM src_associations_src sas
+ JOIN bin_associations_binaries bab ON sas.src = bab.source
+ JOIN suite s on s.id = bab.suite
+ WHERE s.suite_name = :suite_name AND bab.package = :package
+ GROUP BY sas.source, bab.package
+ ORDER BY sas.source, bab.package"""
args = { 'suite_name': suite_name, 'package': package }
return session.execute(query, args)

diff --git a/dak/dakdb/update27.py b/dak/dakdb/update27.py
index ee8f584..b06932b 100755
--- a/dak/dakdb/update27.py
+++ b/dak/dakdb/update27.py
@@ -50,53 +50,6 @@ CREATE OR REPLACE VIEW bin_associations_binaries AS
JOIN binaries ON bin_associations.bin = binaries.id;
""")

- print "Drop old views."
- c.execute("DROP VIEW IF EXISTS source_suite_unique CASCADE")
- c.execute("DROP VIEW IF EXISTS obsolete_source CASCADE")
- c.execute("DROP VIEW IF EXISTS source_bin CASCADE")
- c.execute("DROP VIEW IF EXISTS newest_source_bab CASCADE")
-
- print "Create new views."
- # returns source package names from suite without duplicates;
- # rationale: cruft-report and rm cannot handle duplicates (yet)
- c.execute("""
-CREATE VIEW source_suite_unique
- AS SELECT source, suite
- FROM source_suite GROUP BY source, suite HAVING count(*) = 1;
- """)
- # returns obsolete sources without binaries in the same suite;
- # outputs install_date to detect source only (or binary throw away)
- # uploads; duplicates are skipped
- c.execute("""
-CREATE VIEW obsolete_source
- AS SELECT ss.src, ss.source, ss.version, ss.suite,
- to_char(ss.install_date, 'YYYY-MM-DD') AS install_date
- FROM source_suite ss
- JOIN source_suite_unique ssu
- ON ss.source = ssu.source AND ss.suite = ssu.suite
- LEFT JOIN bin_associations_binaries bab
- ON ss.src = bab.source AND ss.suite = bab.suite
- WHERE bab.id IS NULL;
- """)
- # returns source package names and its binaries from any suite
- c.execute("""
-CREATE VIEW source_bin
- AS SELECT b.package, MAX(b.version) AS version, sas.source
- FROM binaries b
- JOIN src_associations_src sas
- ON b.source = sas.src
- GROUP BY b.package, sas.source
- """)
- # returns binaries from suite and their source with max(version)
- # grouped by source name, binary name, and suite
- c.execute("""
-CREATE VIEW newest_source_bab
- AS SELECT sas.source, MAX(sas.version) AS srcver, bab.package, bab.suite
- FROM src_associations_src sas
- JOIN bin_associations_binaries bab ON sas.src = bab.source
- GROUP BY sas.source, bab.package, bab.suite;
- """)
-
print "Grant permissions to views."
c.execute("GRANT SELECT ON binfiles_suite_component_arch TO PUBLIC;");
c.execute("GRANT SELECT ON srcfiles_suite_component TO PUBLIC;");
@@ -116,10 +69,6 @@ CREATE VIEW newest_source_bab
c.execute("GRANT SELECT ON src_associations_bin TO PUBLIC;");
c.execute("GRANT SELECT ON almost_obsolete_all_associations TO PUBLIC;");
c.execute("GRANT SELECT ON obsolete_all_associations TO PUBLIC;");
- c.execute("GRANT SELECT ON source_suite_unique TO PUBLIC;");
- c.execute("GRANT SELECT ON obsolete_source TO PUBLIC;");
- c.execute("GRANT SELECT ON source_bin TO PUBLIC;");
- c.execute("GRANT SELECT ON newest_source_bab TO PUBLIC;");

print "Committing"
c.execute("UPDATE config SET value = '27' WHERE name = 'db_revision'")
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 1/2] cruft-report: reimplement 'obsolete source'

2009-11-28T14:12:56Z

Signed-off-by: Torsten Werner <twerner@...>
---
dak/cruft_report.py | 109 +++++++++++++++++++++++-----------------
dak/dakdb/update27.py | 131 +++++++++++++++++++++++++++++++++++++++++++++++++
dak/update_db.py | 2 +-
3 files changed, 194 insertions(+), 48 deletions(-)
create mode 100755 dak/dakdb/update27.py

diff --git a/dak/cruft_report.py b/dak/cruft_report.py
index 73619d5..384ad0e 100755
--- a/dak/cruft_report.py
+++ b/dak/cruft_report.py
@@ -272,47 +272,64 @@ def do_dubious_nbs(dubious_nbs):

################################################################################

-def do_obsolete_source(duplicate_bins, bin2source):
- obsolete = {}
- for key in duplicate_bins.keys():
- (source_a, source_b) = key.split('_')
- for source in [ source_a, source_b ]:
- if not obsolete.has_key(source):
- if not source_binaries.has_key(source):
- # Source has already been removed
- continue
- else:
- obsolete[source] = [ i.strip() for i in source_binaries[source].split(',') ]
- for binary in duplicate_bins[key]:
- if bin2source.has_key(binary) and bin2source[binary]["source"] == source:
- continue
- if binary in obsolete[source]:
- obsolete[source].remove(binary)
-
- to_remove = []
- output = "Obsolete source package\n"
- output += "-----------------------\n\n"
- obsolete_keys = obsolete.keys()
- obsolete_keys.sort()
- for source in obsolete_keys:
- if not obsolete[source]:
- to_remove.append(source)
- output += " * %s (%s)\n" % (source, source_versions[source])
- for binary in [ i.strip() for i in source_binaries[source].split(',') ]:
- if bin2source.has_key(binary):
- output += " o %s (%s) is built by %s.\n" \
- % (binary, bin2source[binary]["version"],
- bin2source[binary]["source"])
- else:
- output += " o %s is not built.\n" % binary
- output += "\n"
-
- if to_remove:
- print output
-
- print "Suggested command:"
- print " dak rm -S -p -m \"[auto-cruft] obsolete source package\" %s" % (" ".join(to_remove))
- print
+def obsolete_source(suite_name, session):
+ """returns obsolete source packages for suite_name sorted by
+ install_date"""
+
+ query = """
+SELECT os.src, os.source, os.version, os.install_date
+ FROM obsolete_source os
+ JOIN suite s on s.id = os.suite
+ WHERE s.suite_name = :suite_name
+ ORDER BY install_date"""
+ args = { 'suite_name': suite_name }
+ return session.execute(query, args)
+
+def source_bin(source, session):
+ """returns binaries built by source for all or no suite"""
+
+ query = """
+SELECT package
+ FROM source_bin
+ WHERE source = :source
+ ORDER BY package"""
+ args = { 'source': source }
+ return session.execute(query, args)
+
+def newest_source_bab(suite_name, package, session):
+ """returns newest source that builds binary package in suite"""
+
+ query = """
+SELECT source, srcver
+ FROM newest_source_bab nsb
+ JOIN suite s on s.id = nsb.suite
+ WHERE s.suite_name = :suite_name AND nsb.package = :package
+ ORDER BY source"""
+ args = { 'suite_name': suite_name, 'package': package }
+ return session.execute(query, args)
+
+def report_obsolete_source(suite_name, session):
+ rows = obsolete_source(suite_name, session)
+ if rows.rowcount == 0:
+ return
+ print \
+"""Obsolete source packages in suite %s
+----------------------------------%s\n""" % \
+ (suite_name, '-' * len(suite_name))
+ for os_row in rows.fetchall():
+ (src, old_source, version, install_date) = os_row
+ print " * obsolete source %s version %s installed at %s" % \
+ (old_source, version, install_date)
+ for sb_row in source_bin(old_source, session):
+ (package, ) = sb_row
+ print " - has built binary %s" % package
+ for nsb_row in newest_source_bab(suite_name, package, session):
+ (new_source, srcver) = nsb_row
+ print " currently built by source %s version %s" % \
+ (new_source, srcver)
+ print " - suggested command:"
+ rm_opts = "-S -p -m \"[auto-cruft] obsolete source package\""
+ print " dak rm -s %s %s %s\n" % (suite_name, rm_opts, old_source)

def get_suite_binaries(suite, session):
# Initalize a large hash table of all binary packages
@@ -358,9 +375,7 @@ def main ():

# Set up checks based on mode
if Options["Mode"] == "daily":
- checks = [ "nbs", "nviu", "nvit" ]
- # 'obsolete source' is broken since the introduction of dak dominate
- #checks = [ "nbs", "nviu", "nvit", "obsolete source" ]
+ checks = [ "nbs", "nviu", "nvit", "obsolete source" ]
elif Options["Mode"] == "full":
checks = [ "nbs", "nviu", "nvit", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ]
else:
@@ -385,6 +400,9 @@ def main ():
suite_id = suite.suite_id
suite_name = suite.suite_name.lower()

+ if "obsolete source" in checks:
+ report_obsolete_source(suite_name, session)
+
bin_not_built = {}

if "bnb" in checks:
@@ -503,9 +521,6 @@ def main ():
packages.close()
os.unlink(temp_filename)

- if "obsolete source" in checks:
- do_obsolete_source(duplicate_bins, bin2source)
-
# Distinguish dubious (version numbers match) and 'real' NBS (they don't)
dubious_nbs = {}
real_nbs = {}
diff --git a/dak/dakdb/update27.py b/dak/dakdb/update27.py
new file mode 100755
index 0000000..ee8f584
--- /dev/null
+++ b/dak/dakdb/update27.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+
+"""
+Add views for new obsolete source detection.
+
+@contact: Debian FTP Master <ftpmaster@...>
+@copyright: 2009 Torsten Werner <twerner@...>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import psycopg2
+
+def do_update(self):
+ print "Add/modify views for obsolete source detection."
+
+ try:
+ c = self.db.cursor()
+
+ print "Replace old views."
+ # joins src_associations and source
+ c.execute("""
+CREATE OR REPLACE VIEW source_suite AS
+ SELECT src_associations.id, source.id AS src, source.source, source.version,
+ src_associations.suite, suite.suite_name, source.install_date
+ FROM source
+ JOIN src_associations ON source.id = src_associations.source
+ JOIN suite ON suite.id = src_associations.suite;
+ """)
+ # joins bin_associations and binaries
+ c.execute("""
+CREATE OR REPLACE VIEW bin_associations_binaries AS
+ SELECT bin_associations.id, bin_associations.bin, binaries.package,
+ binaries.version, bin_associations.suite, binaries.architecture,
+ binaries.source
+ FROM bin_associations
+ JOIN binaries ON bin_associations.bin = binaries.id;
+ """)
+
+ print "Drop old views."
+ c.execute("DROP VIEW IF EXISTS source_suite_unique CASCADE")
+ c.execute("DROP VIEW IF EXISTS obsolete_source CASCADE")
+ c.execute("DROP VIEW IF EXISTS source_bin CASCADE")
+ c.execute("DROP VIEW IF EXISTS newest_source_bab CASCADE")
+
+ print "Create new views."
+ # returns source package names from suite without duplicates;
+ # rationale: cruft-report and rm cannot handle duplicates (yet)
+ c.execute("""
+CREATE VIEW source_suite_unique
+ AS SELECT source, suite
+ FROM source_suite GROUP BY source, suite HAVING count(*) = 1;
+ """)
+ # returns obsolete sources without binaries in the same suite;
+ # outputs install_date to detect source only (or binary throw away)
+ # uploads; duplicates are skipped
+ c.execute("""
+CREATE VIEW obsolete_source
+ AS SELECT ss.src, ss.source, ss.version, ss.suite,
+ to_char(ss.install_date, 'YYYY-MM-DD') AS install_date
+ FROM source_suite ss
+ JOIN source_suite_unique ssu
+ ON ss.source = ssu.source AND ss.suite = ssu.suite
+ LEFT JOIN bin_associations_binaries bab
+ ON ss.src = bab.source AND ss.suite = bab.suite
+ WHERE bab.id IS NULL;
+ """)
+ # returns source package names and its binaries from any suite
+ c.execute("""
+CREATE VIEW source_bin
+ AS SELECT b.package, MAX(b.version) AS version, sas.source
+ FROM binaries b
+ JOIN src_associations_src sas
+ ON b.source = sas.src
+ GROUP BY b.package, sas.source
+ """)
+ # returns binaries from suite and their source with max(version)
+ # grouped by source name, binary name, and suite
+ c.execute("""
+CREATE VIEW newest_source_bab
+ AS SELECT sas.source, MAX(sas.version) AS srcver, bab.package, bab.suite
+ FROM src_associations_src sas
+ JOIN bin_associations_binaries bab ON sas.src = bab.source
+ GROUP BY sas.source, bab.package, bab.suite;
+ """)
+
+ print "Grant permissions to views."
+ c.execute("GRANT SELECT ON binfiles_suite_component_arch TO PUBLIC;");
+ c.execute("GRANT SELECT ON srcfiles_suite_component TO PUBLIC;");
+ c.execute("GRANT SELECT ON binaries_suite_arch TO PUBLIC;");
+ c.execute("GRANT SELECT ON newest_all_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON obsolete_any_by_all_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON newest_any_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON obsolete_any_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON source_suite TO PUBLIC;");
+ c.execute("GRANT SELECT ON newest_source TO PUBLIC;");
+ c.execute("GRANT SELECT ON newest_src_association TO PUBLIC;");
+ c.execute("GRANT SELECT ON any_associations_source TO PUBLIC;");
+ c.execute("GRANT SELECT ON src_associations_src TO PUBLIC;");
+ c.execute("GRANT SELECT ON almost_obsolete_src_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON obsolete_src_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON bin_associations_binaries TO PUBLIC;");
+ c.execute("GRANT SELECT ON src_associations_bin TO PUBLIC;");
+ c.execute("GRANT SELECT ON almost_obsolete_all_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON obsolete_all_associations TO PUBLIC;");
+ c.execute("GRANT SELECT ON source_suite_unique TO PUBLIC;");
+ c.execute("GRANT SELECT ON obsolete_source TO PUBLIC;");
+ c.execute("GRANT SELECT ON source_bin TO PUBLIC;");
+ c.execute("GRANT SELECT ON newest_source_bab TO PUBLIC;");
+
+ print "Committing"
+ c.execute("UPDATE config SET value = '27' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.InternalError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Database error, rollback issued. Error message : %s" % (str(msg))
+
diff --git a/dak/update_db.py b/dak/update_db.py
index e7b1984..5d1cb86 100755
--- a/dak/update_db.py
+++ b/dak/update_db.py
@@ -45,7 +45,7 @@ from daklib.dak_exceptions import DBUpdateError
################################################################################

Cnf = None
-required_database_schema = 26
+required_database_schema = 27

################################################################################

--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master] fix OverrideSuite handling

2009-11-28T10:34:31Z

do it in determine_new() (queue.py) instead of do_new() (process_new.py)

Signed-off-by: Torsten Werner <twerner@...>
---
dak/process_new.py | 15 ---------------
daklib/queue.py | 16 ++++++++++++++++
2 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/dak/process_new.py b/dak/process_new.py
index 060fa23..894d07d 100755
--- a/dak/process_new.py
+++ b/dak/process_new.py
@@ -611,21 +611,6 @@ def do_new(upload, session):
# Make a copy of distribution we can happily trample on
changes["suite"] = copy.copy(changes["distribution"])

- # Fix up the list of target suites
- for suite in changes["suite"].keys():
- override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
- if override:
- (olderr, newerr) = (get_suite(suite, session) == None,
- get_suite(override, session) == None)
- if olderr or newerr:
- (oinv, newinv) = ("", "")
- if olderr: oinv = "invalid "
- if newerr: ninv = "invalid "
- print "warning: overriding %ssuite %s to %ssuite %s" % (
- oinv, suite, ninv, override)
- del changes["suite"][suite]
- changes["suite"][override] = 1
-
# The main NEW processing loop
done = 0
while not done:
diff --git a/daklib/queue.py b/daklib/queue.py
index 5ea545f..0433a96 100755
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -148,6 +148,22 @@ def determine_new(changes, files, warn=1):
if f.has_key("othercomponents"):
new[pkg]["othercomponents"] = f["othercomponents"]

+ # Fix up the list of target suites
+ cnf = Config()
+ for suite in changes["suite"].keys():
+ override = cnf.Find("Suite::%s::OverrideSuite" % (suite))
+ if override:
+ (olderr, newerr) = (get_suite(suite, session) == None,
+ get_suite(override, session) == None)
+ if olderr or newerr:
+ (oinv, newinv) = ("", "")
+ if olderr: oinv = "invalid "
+ if newerr: ninv = "invalid "
+ print "warning: overriding %ssuite %s to %ssuite %s" % (
+ oinv, suite, ninv, override)
+ del changes["suite"][suite]
+ changes["suite"][override] = 1
+
for suite in changes["suite"].keys():
for pkg in new.keys():
ql = get_override(pkg, suite, new[pkg]["component"], new[pkg]["type"], session)
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master] Adjust regex

2009-11-25T15:17:53Z

Match "Archive maintenance timestamp" only if present at the
beginning of the row, thus ignoring bogus messages eventually registered
in the logs.

Signed-off-by: Luca Falavigna <dktrkranz@...>
---
tools/logs.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/tools/logs.py b/tools/logs.py
index c4eb6be..6db78d9 100755
--- a/tools/logs.py
+++ b/tools/logs.py
@@ -46,7 +46,7 @@ if m:
raise Exception("I don't like command line arguments including char '%s'"%m.group(0))

if args:
- for l in os.popen('bzgrep -H "Archive maintenance timestamp" "'+'" "'.join(args)+'"'):
+ for l in os.popen('bzgrep -H "^Archive maintenance timestamp" "'+'" "'.join(args)+'"'):
m = LINE.match(l)
if not m:
raise Exception("woops '%s'"%l)
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master] different tags

2009-11-25T15:12:32Z

Signed-off-by: Joerg Jaspert <joerg@...>
---
tools/logs.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/tools/logs.py b/tools/logs.py
index c4eb6be..182cdbd 100755
--- a/tools/logs.py
+++ b/tools/logs.py
@@ -14,7 +14,7 @@ LINE = re.compile(r'(?:|.*/)dinstall_(\d{4})\.(\d{2})\.(\d{2})-(\d{2}):(\d{2}):(
r'Archive maintenance timestamp $([^$]*)\): (\d{2}):(\d{2}):(\d{2})$')
UNSAFE = re.compile(r'[^a-zA-Z/\._:0-9\- ]')

-graphs = {"dinstall1": {"keystolist":["pg_dump1", "i18n 1", "accepted", "make-suite-file-list", "apt-ftparchive",
+graphs = {"dinstall1": {"keystolist":["pg_dump1", "i18n 1", "accepted", "dominate", "generate-filelist", "apt-ftparchive",
"pdiff", "release files", "w-b", "i18n 2", "apt-ftparchive cleanup"],
"showothers":True},
"dinstall2": {"keystolist":['External Updates', 'p-u-new', 'o-p-u-new', 'cruft', 'import-keyring', 'overrides', 'cleanup', 'scripts', 'mirror hardlinks', 'stats', 'compress', "pkg-file-mapping" ],
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master] Do not break manual rejects when a note exists

2009-11-25T13:21:48Z

Signed-off-by: Joerg Jaspert <joerg@...>
---
dak/process_new.py | 2 +-
daklib/queue.py | 9 +++++----
2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/dak/process_new.py b/dak/process_new.py
index 7841a7d..060fa23 100755
--- a/dak/process_new.py
+++ b/dak/process_new.py
@@ -679,7 +679,7 @@ def do_new(upload, session):
elif answer == 'M' and not Options["Trainee"]:
aborted = upload.do_reject(manual=1,
reject_message=Options["Manual-Reject"],
- note=get_new_comments(changes.get("source", ""), session=session))
+ notes=get_new_comments(changes.get("source", ""), session=session))
if not aborted:
upload.pkg.remove_known_changes(session=session)
session.commit()
diff --git a/daklib/queue.py b/daklib/queue.py
index c9a6532..5ea545f 100755
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -2061,7 +2061,7 @@ distribution."""
os.close(dest_fd)

###########################################################################
- def do_reject (self, manual=0, reject_message="", note=""):
+ def do_reject (self, manual=0, reject_message="", notes=""):
"""
Reject an upload. If called without a reject message or C{manual} is
true, spawn an editor so the user can write one.
@@ -2080,9 +2080,10 @@ distribution."""
if manual and not reject_message:
(fd, temp_filename) = utils.temp_filename()
temp_file = os.fdopen(fd, 'w')
- if len(note) > 0:
- for line in note:
- temp_file.write(line)
+ if len(notes) > 0:
+ for note in notes:
+ temp_file.write("\nAuthor: %s\nVersion: %s\nTimestamp: %s\n\n%s" \
+ % (note.author, note.version, note.notedate, note.comment))
temp_file.close()
editor = os.environ.get("EDITOR","vi")
answer = 'E'
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 1/2] p-n

2009-11-25T13:05:44Z

call upload.remove_known_changes in a few more places.
also give it our session.

Signed-off-by: Joerg Jaspert <joerg@...>
---
dak/process_new.py | 4 +++-
1 files changed, 3 insertions(+), 1 deletions(-)

diff --git a/dak/process_new.py b/dak/process_new.py
index 6bdd301..b5fd912 100755
--- a/dak/process_new.py
+++ b/dak/process_new.py
@@ -95,6 +95,7 @@ def recheck(upload, session):

if answer == 'R':
upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
+ upload.pkg.remove_known_changes(session=session)
return 0
elif answer == 'S':
return 0
@@ -679,7 +680,7 @@ def do_new(upload, session):
reject_message=Options["Manual-Reject"],
note=get_new_comments(changes.get("source", ""), session=session))
if not aborted:
- upload.pkg.remove_known_changes()
+ upload.pkg.remove_known_changes(session=session)
Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
done = 1
elif answer == 'N':
@@ -769,6 +770,7 @@ def do_byhand(upload, session):
elif answer == 'M':
Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
+ upload.pkg.remove_known_changes(session=session)
done = 1
elif answer == 'S':
done = 1
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 2/2] p-n

2009-11-25T13:05:44Z

make sure to commit after we rejected stuff and changed database.

Signed-off-by: Joerg Jaspert <joerg@...>
---
dak/process_new.py | 3 +++
1 files changed, 3 insertions(+), 0 deletions(-)

diff --git a/dak/process_new.py b/dak/process_new.py
index b5fd912..7841a7d 100755
--- a/dak/process_new.py
+++ b/dak/process_new.py
@@ -96,6 +96,7 @@ def recheck(upload, session):
if answer == 'R':
upload.do_reject(manual=0, reject_message='\n'.join(upload.rejects))
upload.pkg.remove_known_changes(session=session)
+ session.commit()
return 0
elif answer == 'S':
return 0
@@ -681,6 +682,7 @@ def do_new(upload, session):
note=get_new_comments(changes.get("source", ""), session=session))
if not aborted:
upload.pkg.remove_known_changes(session=session)
+ session.commit()
Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
done = 1
elif answer == 'N':
@@ -771,6 +773,7 @@ def do_byhand(upload, session):
Logger.log(["BYHAND REJECT: %s" % (upload.pkg.changes_file)])
upload.do_reject(manual=1, reject_message=Options["Manual-Reject"])
upload.pkg.remove_known_changes(session=session)
+ session.commit()
done = 1
elif answer == 'S':
done = 1
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master] import-users

2009-11-25T12:57:26Z

delete no longer existing users instead of just needlessly spamming us
with their names

Signed-off-by: Joerg Jaspert <joerg@...>
---
dak/import_users_from_passwd.py | 5 +++--
1 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/dak/import_users_from_passwd.py b/dak/import_users_from_passwd.py
index 07c6193..9584867 100755
--- a/dak/import_users_from_passwd.py
+++ b/dak/import_users_from_passwd.py
@@ -99,8 +99,9 @@ def main ():
keys = postgres_unames.keys()
keys.sort()
for uname in keys:
- if not passwd_unames.has_key(uname)and not known_postgres_unames.has_key(uname):
- print "W: %s is in Postgres but not the passwd file or list of known Postgres users." % (uname)
+ if not passwd_unames.has_key(uname) and not known_postgres_unames.has_key(uname):
+ print "I: Deleting %s from Postgres, no longer in passwd or list of known Postgres users" % (uname)
+ q = session.execute('DROP USER "%s"' % (uname))

keys = passwd_unames.keys()
keys.sort()
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 1/2] Handle broken pipe while waiting for action

2009-11-24T15:04:42Z

our_raw_input waits for action, but an exception is raised if user
does not provide it. Loop until a correct action is taken, this way
dak does not horribly crash each time.

Signed-off-by: Luca Falavigna <dktrkranz@...>
---
daklib/utils.py | 7 ++++++-
1 files changed, 6 insertions(+), 1 deletions(-)

diff --git a/daklib/utils.py b/daklib/utils.py
index 3cc4053..cfbf8aa 100644
--- a/daklib/utils.py
+++ b/daklib/utils.py
@@ -115,7 +115,12 @@ def open_file(filename, mode='r'):

def our_raw_input(prompt=""):
if prompt:
- sys.stdout.write(prompt)
+ while 1:
+ try:
+ sys.stdout.write(prompt)
+ break
+ except IOError:
+ pass
sys.stdout.flush()
try:
ret = raw_input()
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 2/2] web: add link to cruft-report-daily.txt

2009-11-24T15:04:42Z

Signed-off-by: Torsten Werner <twerner@...>
---
web/index.html | 7 +++++++
1 files changed, 7 insertions(+), 0 deletions(-)

diff --git a/web/index.html b/web/index.html
index 9f0adca..8b42060 100644
--- a/web/index.html
+++ b/web/index.html
@@ -41,6 +41,7 @@
<li><a href="#new">NEW</a></li>
<li><a href="#pending">Pending removals</a></li>
<li><a href="#removed">Removed packages</a></li>
+ <li><a href="#cruft">Cruft Report</a></li>
<li><a href="#testing">Testing</a></li>
<li><a href="#stable">Stable</a></li>
<li><a href="#rejections">Rejections</a></li>
@@ -161,6 +162,12 @@
</p>
</div>

+ <div id="cruft">
+ <h1>Cruft Report</h1>
+ <p>Some packages which needs to be removed manually are found in
+ <a href="cruft-report-daily.txt">the cruft-report</a>.</p>
+ </div>
+
<div id="testing">
<h1>Testing</h1>
<p>Squeeze is testing, sid is unstable. For more details please look
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 2/3] Fix indentation on commit to be outside for loop but inside try. Add update to db_revision to set it to 26.

2009-11-24T14:17:51Z

---
dak/dakdb/update26.py | 5 +++--
1 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/dak/dakdb/update26.py b/dak/dakdb/update26.py
index 70ad62b..3c7b082 100755
--- a/dak/dakdb/update26.py
+++ b/dak/dakdb/update26.py
@@ -57,8 +57,9 @@ def do_update(self):
c.execute("""CREATE TRIGGER modified_%s BEFORE UPDATE ON %s
FOR EACH ROW EXECUTE PROCEDURE tfunc_set_modified()""" % (updatetable, updatetable))

- print "Committing"
- self.db.commit()
+ print "Committing"
+ c.execute("UPDATE config SET value = '26' WHERE name = 'db_revision'")
+ self.db.commit()

except psycopg2.InternalError, msg:
self.db.rollback()
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 3/3] Update update_db to use schema 26

2009-11-24T14:17:51Z

---
dak/update_db.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/dak/update_db.py b/dak/update_db.py
index 50b293f..e7b1984 100755
--- a/dak/update_db.py
+++ b/dak/update_db.py
@@ -45,7 +45,7 @@ from daklib.dak_exceptions import DBUpdateError
################################################################################

Cnf = None
-required_database_schema = 25
+required_database_schema = 26

################################################################################

--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 1/3] Add dakdb/update26.py to add created and modified fields on all tables and add a trigger to update the modified field on update

2009-11-24T14:17:51Z

---
dak/dakdb/update26.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 66 insertions(+), 0 deletions(-)
create mode 100755 dak/dakdb/update26.py

diff --git a/dak/dakdb/update26.py b/dak/dakdb/update26.py
new file mode 100755
index 0000000..70ad62b
--- /dev/null
+++ b/dak/dakdb/update26.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+
+"""
+Add created,modified columns for all tables.
+
+@contact: Debian FTP Master <ftpmaster@...>
+@copyright: 2009 Barry deFreese <bdefreese@...>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import psycopg2
+
+def do_update(self):
+ print "Add created, modified fields for all tables."
+
+ updatetables = ['architecture', 'archive', 'bin_associations', 'bin_contents',
+ 'binaries', 'binary_acl', 'binary_acl_map', 'build_queue', 'build_queue_files',
+ 'changes', 'changes_pending_binaries', 'changes_pending_files',
+ 'changes_pending_files_map', 'changes_pending_source', 'changes_pending_source_files',
+ 'changes_pool_files', 'component', 'config', 'dsc_files', 'files', 'fingerprint',
+ 'keyring_acl_map', 'keyrings', 'location', 'maintainer', 'new_comments', 'override',
+ 'override_type', 'policy_queue', 'priority', 'section', 'source', 'source_acl',
+ 'src_associations', 'src_format', 'src_uploaders', 'suite', 'suite_architectures',
+ 'suite_build_queue_copy', 'suite_src_formats', 'uid', 'upload_blocks']
+
+ c = self.db.cursor()
+
+ print "Create trigger function."
+ c.execute("""CREATE OR REPLACE FUNCTION tfunc_set_modified() RETURNS trigger AS $$
+ BEGIN NEW.modified = now(); return NEW; END;
+ $$ LANGUAGE 'plpgsql'""")
+
+ try:
+ for updatetable in updatetables:
+
+ print "Add created field to %s." % updatetable
+ c.execute("ALTER TABLE %s ADD COLUMN created TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()" % updatetable)
+
+ print "Add modified field to %s." % updatetable
+ c.execute("ALTER TABLE %s ADD COLUMN modified TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()" % updatetable)
+
+ print "Create modified trigger."
+ c.execute("""CREATE TRIGGER modified_%s BEFORE UPDATE ON %s
+ FOR EACH ROW EXECUTE PROCEDURE tfunc_set_modified()""" % (updatetable, updatetable))
+
+ print "Committing"
+ self.db.commit()
+
+ except psycopg2.InternalError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Database error, rollback issued. Error message : %s" % (str(msg))
+
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master] Cnf[...] changed to cnf[...] in override.py

2009-11-24T00:12:23Z

---
dak/override.py | 6 +++---
1 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/dak/override.py b/dak/override.py
index e253967..ce35d92 100755
--- a/dak/override.py
+++ b/dak/override.py
@@ -240,9 +240,9 @@ def main ():
Subst["__BCC__"] = "Bcc: " + ", ".join(bcc)
else:
Subst["__BCC__"] = "X-Filler: 42"
- Subst["__CC__"] = "Cc: " + package + "@" + Cnf["Dinstall::PackagesServer"] + "\nX-DAK: dak override"
- Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"]
- Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"]
+ Subst["__CC__"] = "Cc: " + package + "@" + cnf["Dinstall::PackagesServer"] + "\nX-DAK: dak override"
+ Subst["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"]
+ Subst["__DISTRO__"] = cnf["Dinstall::MyDistribution"]
Subst["__WHOAMI__"] = utils.whoami()
Subst["__SOURCE__"] = package

--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[PATCH] Add SIG_SUBPACKET as a known_keyword.

2009-11-23T07:22:23Z

SIG_SUBPACKET denotes a preferred keyserver was set for the key but requires no
additional processing and has no bearing on signature validity. It simply needs
to be ignored.

Signed-off-by: Jeremy T. Bouse <jbouse@...>
---
daklib/utils.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/daklib/utils.py b/daklib/utils.py
index 3cc4053..7d5c4c4 100644
--- a/daklib/utils.py
+++ b/daklib/utils.py
@@ -1360,7 +1360,7 @@ def check_signature (sig_filename, data_filename="", keyrings=None, autofetch=No

# Finally ensure there's not something we don't recognise
known_keywords = dict(VALIDSIG="",SIG_ID="",GOODSIG="",BADSIG="",ERRSIG="",
- SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",
+ SIGEXPIRED="",KEYREVOKED="",NO_PUBKEY="",BADARMOR="",SIG_SUBPACKET="",
NODATA="",NOTATION_DATA="",NOTATION_NAME="",KEYEXPIRED="",POLICY_URL="")

for keyword in keywords.keys():
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master] p-n

2009-11-22T15:39:01Z

only remove known changes if reject isnt aborted

Signed-off-by: Joerg Jaspert <joerg@...>
---
dak/process_new.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/dak/process_new.py b/dak/process_new.py
index 9f7694d..6bdd301 100755
--- a/dak/process_new.py
+++ b/dak/process_new.py
@@ -675,11 +675,11 @@ def do_new(upload, session):
elif answer == 'E' and not Options["Trainee"]:
new = edit_overrides (new, upload, session)
elif answer == 'M' and not Options["Trainee"]:
- upload.pkg.remove_known_changes()
aborted = upload.do_reject(manual=1,
reject_message=Options["Manual-Reject"],
note=get_new_comments(changes.get("source", ""), session=session))
if not aborted:
+ upload.pkg.remove_known_changes()
Logger.log(["NEW REJECT: %s" % (upload.pkg.changes_file)])
done = 1
elif answer == 'N':
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 2/2] no longer changes, its self.pkg.changes here

2009-11-22T14:53:33Z

Signed-off-by: Joerg Jaspert <joerg@...>
---
daklib/queue.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/daklib/queue.py b/daklib/queue.py
index 33310da..c9a6532 100755
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -427,7 +427,7 @@ class Upload(object):
self.pkg.changes["changedbyemail"] = ""

self.rejects.append("%s: Changed-By field ('%s') failed to parse: %s" \
- % (filename, changes["changed-by"], msg))
+ % (filename, self.pkg.changes["changed-by"], msg))

# Ensure all the values in Closes: are numbers
if self.pkg.changes.has_key("closes"):
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 1/2] its self.rejects

2009-11-22T14:53:33Z

Signed-off-by: Joerg Jaspert <joerg@...>
---
daklib/queue.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/daklib/queue.py b/daklib/queue.py
index 67e6563..33310da 100755
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -2307,7 +2307,7 @@ distribution."""
cansave = 1

if not cansave:
- self.reject.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))
+ self.rejects.append("%s: old version (%s) in %s <= new version (%s) targeted at %s." % (filename, existent_version, suite, new_version, target_suite))

################################################################################
def check_binary_against_db(self, filename, session):
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master] Remove suite validation and use check_distributions instead and move it further up in process

2009-11-22T08:48:44Z

---
dak/process_new.py | 7 +++----
1 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/dak/process_new.py b/dak/process_new.py
index 4b2cff3..9f7694d 100755
--- a/dak/process_new.py
+++ b/dak/process_new.py
@@ -603,6 +603,9 @@ def do_new(upload, session):
changes = upload.pkg.changes
cnf = Config()

+ # Check for a valid distribution
+ upload.check_distributions()
+
# Make a copy of distribution we can happily trample on
changes["suite"] = copy.copy(changes["distribution"])

@@ -620,10 +623,6 @@ def do_new(upload, session):
oinv, suite, ninv, override)
del changes["suite"][suite]
changes["suite"][override] = 1
- # Validate suites
- for suite in changes["suite"].keys():
- if get_suite(suite, session) is None:
- utils.fubar("%s has invalid suite '%s' (possibly overriden). say wha?" % (changes, suite))

# The main NEW processing loop
done = 0
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 2/3] daklib/holding.py: ftptrainees don't have write permissions to holding directory, so allow them to store packages to be reviewed into TempPath.

2009-11-21T05:37:01Z

Signed-off-by: Luca Falavigna <dktrkranz@...>
---
daklib/holding.py | 3 +++
1 files changed, 3 insertions(+), 0 deletions(-)

diff --git a/daklib/holding.py b/daklib/holding.py
index b637738..79d5ef7 100644
--- a/daklib/holding.py
+++ b/daklib/holding.py
@@ -46,6 +46,9 @@ class Holding(object):

self.in_holding = {}
self.holding_dir = Config()["Dir::Queue::Holding"]
+ # ftptrainees haven't access to holding, use a temp directory instead
+ if not os.access(self.holding_dir, os.W_OK):
+ self.holding_dir = Config()["TempPath::TempPath"]

def copy_to_holding(self, filename):
base_filename = os.path.basename(filename)
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 1/3] config/debian/dak.conf, dak/transitions.py:

2009-11-21T05:37:01Z

Move TempPath under its own domain, so it can be used by some other
pieces of dak. Adjust transitions accordingly.

Signed-off-by: Luca Falavigna <dktrkranz@...>
---
config/debian/dak.conf | 6 +++++-
dak/transitions.py | 12 ++++++------
2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/config/debian/dak.conf b/config/debian/dak.conf
index ceafaed..d7006c8 100644
--- a/config/debian/dak.conf
+++ b/config/debian/dak.conf
@@ -49,7 +49,6 @@ Dinstall
Transitions
{
Notifications "debian-devel@...";
- TempPath "/srv/ftp.debian.org/tmp/";
};

Generate-Index-Diffs
@@ -744,3 +743,8 @@ Contents
Header "contents";
Root "/srv/ftp.debian.org/test/";
}
+
+TempPath
+{
+ TempPath "/srv/ftp.debian.org/tmp/";
+};
diff --git a/dak/transitions.py b/dak/transitions.py
index 4c4ac78..2bd3238 100755
--- a/dak/transitions.py
+++ b/dak/transitions.py
@@ -287,8 +287,8 @@ def write_transitions_from_file(from_file):
"""

# Lets check if from_file is in the directory we expect it to be in
- if not os.path.abspath(from_file).startswith(Cnf["Transitions::TempPath"]):
- print "Will not accept transitions file outside of %s" % (Cnf["Transitions::TempPath"])
+ if not os.path.abspath(from_file).startswith(Cnf["TempPath::TempPath"]):
+ print "Will not accept transitions file outside of %s" % (Cnf["TempPath::TempPath"])
sys.exit(3)

if Options["sudo"]:
@@ -318,7 +318,7 @@ def temp_transitions_file(transitions):
sudo-ed script and would be unreadable if it has default mkstemp mode
"""

- (fd, path) = tempfile.mkstemp("", "transitions", Cnf["Transitions::TempPath"])
+ (fd, path) = tempfile.mkstemp("", "transitions", Cnf["TempPath::TempPath"])
os.chmod(path, 0644)
f = open(path, "w")
yaml.dump(transitions, f, default_flow_style=False)
@@ -577,13 +577,13 @@ def main():
(Cnf["Dinstall::Reject::ReleaseTransitions"]))
sys.exit(1)
# Also check if our temp directory is defined and existant
- temppath = Cnf.get("Transitions::TempPath", "")
+ temppath = Cnf.get("TempPath::TempPath", "")
if temppath == "":
- utils.warn("Transitions::TempPath not defined")
+ utils.warn("TempPath::TempPath not defined")
sys.exit(1)
if not os.path.exists(temppath):
utils.warn("Temporary path %s not found." %
- (Cnf["Transitions::TempPath"]))
+ (Cnf["TempPath::TempPath"]))
sys.exit(1)

if Options["import"]:
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 3/3] Move TempPath under Dir option instead of creating a new one

2009-11-21T05:37:01Z

Signed-off-by: Luca Falavigna <dktrkranz@...>
---
config/debian/dak.conf | 6 +-----
dak/transitions.py | 12 ++++++------
daklib/holding.py | 2 +-
3 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/config/debian/dak.conf b/config/debian/dak.conf
index d7006c8..bb7682b 100644
--- a/config/debian/dak.conf
+++ b/config/debian/dak.conf
@@ -554,6 +554,7 @@ Dir
Override "/srv/ftp.debian.org/scripts/override/";
QueueBuild "/srv/incoming.debian.org/buildd/";
UrgencyLog "/srv/release.debian.org/britney/input/urgencies/";
+ TempPath "/srv/ftp.debian.org/tmp/";
Queue
{
Byhand "/srv/ftp.debian.org/queue/byhand/";
@@ -743,8 +744,3 @@ Contents
Header "contents";
Root "/srv/ftp.debian.org/test/";
}
-
-TempPath
-{
- TempPath "/srv/ftp.debian.org/tmp/";
-};
diff --git a/dak/transitions.py b/dak/transitions.py
index 2bd3238..157e1c0 100755
--- a/dak/transitions.py
+++ b/dak/transitions.py
@@ -287,8 +287,8 @@ def write_transitions_from_file(from_file):
"""

# Lets check if from_file is in the directory we expect it to be in
- if not os.path.abspath(from_file).startswith(Cnf["TempPath::TempPath"]):
- print "Will not accept transitions file outside of %s" % (Cnf["TempPath::TempPath"])
+ if not os.path.abspath(from_file).startswith(Cnf["Dir::TempPath"]):
+ print "Will not accept transitions file outside of %s" % (Cnf["Dir::TempPath"])
sys.exit(3)

if Options["sudo"]:
@@ -318,7 +318,7 @@ def temp_transitions_file(transitions):
sudo-ed script and would be unreadable if it has default mkstemp mode
"""

- (fd, path) = tempfile.mkstemp("", "transitions", Cnf["TempPath::TempPath"])
+ (fd, path) = tempfile.mkstemp("", "transitions", Cnf["Dir::TempPath"])
os.chmod(path, 0644)
f = open(path, "w")
yaml.dump(transitions, f, default_flow_style=False)
@@ -577,13 +577,13 @@ def main():
(Cnf["Dinstall::Reject::ReleaseTransitions"]))
sys.exit(1)
# Also check if our temp directory is defined and existant
- temppath = Cnf.get("TempPath::TempPath", "")
+ temppath = Cnf.get("Dir::TempPath", "")
if temppath == "":
- utils.warn("TempPath::TempPath not defined")
+ utils.warn("Dir::TempPath not defined")
sys.exit(1)
if not os.path.exists(temppath):
utils.warn("Temporary path %s not found." %
- (Cnf["TempPath::TempPath"]))
+ (Cnf["Dir::TempPath"]))
sys.exit(1)

if Options["import"]:
diff --git a/daklib/holding.py b/daklib/holding.py
index 79d5ef7..f77416c 100644
--- a/daklib/holding.py
+++ b/daklib/holding.py
@@ -48,7 +48,7 @@ class Holding(object):
self.holding_dir = Config()["Dir::Queue::Holding"]
# ftptrainees haven't access to holding, use a temp directory instead
if not os.access(self.holding_dir, os.W_OK):
- self.holding_dir = Config()["TempPath::TempPath"]
+ self.holding_dir = Config()["Dir::TempPath"]

def copy_to_holding(self, filename):
base_filename = os.path.basename(filename)
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 3/5] cruft-report: disable OBS in daily mode

2009-11-19T14:50:48Z

Signed-off-by: Torsten Werner <twerner@...>
---
dak/cruft_report.py | 4 +++-
1 files changed, 3 insertions(+), 1 deletions(-)

diff --git a/dak/cruft_report.py b/dak/cruft_report.py
index cc475f5..4b78487 100755
--- a/dak/cruft_report.py
+++ b/dak/cruft_report.py
@@ -358,7 +358,9 @@ def main ():

# Set up checks based on mode
if Options["Mode"] == "daily":
- checks = [ "nbs", "nviu", "nvit", "obsolete source" ]
+ checks = [ "nbs", "nviu", "nvit" ]
+ # 'obsolete source' is broken since the introduction of dak dominate
+ #checks = [ "nbs", "nviu", "nvit", "obsolete source" ]
elif Options["Mode"] == "full":
checks = [ "nbs", "nviu", "nvit", "obsolete source", "nfu", "dubious nbs", "bnb", "bms", "anais" ]
else:
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 1/5] cruft-report: always check revdeps on NBS removals

2009-11-19T14:50:48Z

Signed-off-by: Torsten Werner <twerner@...>
---
dak/cruft_report.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/dak/cruft_report.py b/dak/cruft_report.py
index 4541bf6..80b2831 100755
--- a/dak/cruft_report.py
+++ b/dak/cruft_report.py
@@ -238,7 +238,7 @@ def do_nbs(real_nbs):
output += " o %s: %s\n" % (version, ", ".join(packages))
if all_packages:
all_packages.sort()
- cmd_output += " dak rm -m \"[auto-cruft] NBS (was built by %s)\" -s %s -b %s\n\n" % (source, suite.suite_name, " ".join(all_packages))
+ cmd_output += " dak rm -m \"[auto-cruft] NBS (was built by %s)\" -s %s -b %s -R\n\n" % (source, suite.suite_name, " ".join(all_packages))

output += "\n"

--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 2/5] cruft-report: ignore duplicate source packages

2009-11-19T14:50:48Z

Signed-off-by: Torsten Werner <twerner@...>
---
dak/cruft_report.py | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dak/cruft_report.py b/dak/cruft_report.py
index 80b2831..cc475f5 100755
--- a/dak/cruft_report.py
+++ b/dak/cruft_report.py
@@ -419,8 +419,8 @@ def main ():

# Check for duplicated packages and build indices for checking "no source" later
source_index = component + '/' + source
- if src_pkgs.has_key(source):
- print " %s is a duplicated source package (%s and %s)" % (source, source_index, src_pkgs[source])
+ #if src_pkgs.has_key(source):
+ # print " %s is a duplicated source package (%s and %s)" % (source, source_index, src_pkgs[source])
src_pkgs[source] = source_index
for binary in binaries_list:
if bin_pkgs.has_key(binary):
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 5/5] queue: fix sponsored uploads

2009-11-19T14:50:48Z

---
daklib/queue.py | 19 +++++++++----------
1 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/daklib/queue.py b/daklib/queue.py
index e39b72d..67e6563 100755
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -1445,16 +1445,15 @@ class Upload(object):
self.check_dm_upload(fpr, session)
else:
# Check source-based permissions for other types
- if self.pkg.changes["architecture"].has_key("source"):
- if fpr.source_acl.access_level is None:
- rej = 'Fingerprint %s may not upload source' % fpr.fingerprint
- rej += '\nPlease contact ftpmaster if you think this is incorrect'
- self.rejects.append(rej)
- return
- else:
- # If not a DM, we allow full upload rights
- uid_email = "%s@..." % (fpr.uid.uid)
- self.check_if_upload_is_sponsored(uid_email, fpr.uid.name)
+ if self.pkg.changes["architecture"].has_key("source") and \
+ fpr.source_acl.access_level is None:
+ rej = 'Fingerprint %s may not upload source' % fpr.fingerprint
+ rej += '\nPlease contact ftpmaster if you think this is incorrect'
+ self.rejects.append(rej)
+ return
+ # If not a DM, we allow full upload rights
+ uid_email = "%s@..." % (fpr.uid.uid)
+ self.check_if_upload_is_sponsored(uid_email, fpr.uid.name)

# Check binary upload permissions
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 4/5] cruft-report: fix exception in full mode

2009-11-19T14:50:48Z

Signed-off-by: Torsten Werner <twerner@...>
---
dak/cruft_report.py | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/dak/cruft_report.py b/dak/cruft_report.py
index 4b78487..73619d5 100755
--- a/dak/cruft_report.py
+++ b/dak/cruft_report.py
@@ -388,7 +388,7 @@ def main ():
bin_not_built = {}

if "bnb" in checks:
- bins_in_suite = get_suite_binaries(suite_name, session)
+ bins_in_suite = get_suite_binaries(suite, session)

# Checks based on the Sources files
components = cnf.ValueList("Suite::%s::Components" % (suite_name))
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master] Really replace msfl by dominate in dinstall

2009-11-18T23:53:28Z

Signed-off-by: Torsten Werner <twerner@...>
---
config/debian/dinstall.functions | 6 +++---
1 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/config/debian/dinstall.functions b/config/debian/dinstall.functions
index ddce969..26c1c8b 100644
--- a/config/debian/dinstall.functions
+++ b/config/debian/dinstall.functions
@@ -129,9 +129,9 @@ function cruft() {
dak check-overrides
}

-function msfl() {
- log "Generating suite file lists for apt-ftparchive"
- dak make-suite-file-list
+function dominate() {
+ log "Removing obsolete source and binary associations"
+ dak dominate
}

function filelist() {
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 3/3] NEW

2009-11-18T14:03:44Z

dont fillin automated reject message when doing a manual NEW reject.

Signed-off-by: Joerg Jaspert <joerg@...>
---
daklib/queue.py | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/daklib/queue.py b/daklib/queue.py
index c7e66a4..e39b72d 100755
--- a/daklib/queue.py
+++ b/daklib/queue.py
@@ -2139,6 +2139,7 @@ distribution."""
user_email_address = utils.whoami() + " <%s>" % (cnf["Dinstall::MyAdminAddress"])
self.Subst["__REJECTOR_ADDRESS__"] = user_email_address
self.Subst["__MANUAL_REJECT_MESSAGE__"] = reject_message
+ self.Subst["__REJECT_MESSAGE__"] = ""
self.Subst["__CC__"] = "Cc: " + cnf["Dinstall::MyEmailAddress"]
reject_mail_message = utils.TemplateSubst(self.Subst, rej_template)
# Write the rejection email out as the <foo>.reason file
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 2/3] log ids of deleted rows to aid debugging

2009-11-18T14:03:44Z

Signed-off-by: Torsten Werner <twerner@...>
---
dak/dominate.py | 7 ++++---
1 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/dak/dominate.py b/dak/dominate.py
index 73eee8c..7bb74c6 100755
--- a/dak/dominate.py
+++ b/dak/dominate.py
@@ -35,10 +35,11 @@ def fetch(reason, query, args, session):
for row in session.execute(query, args).fetchall():
(id, package, version, suite_name, architecture) = row
if Options['No-Action']:
- print "Delete %s %s from %s architecture %s (%s)" % \
- (package, version, suite_name, architecture, reason)
+ print "Delete %s %s from %s architecture %s (%s, %d)" % \
+ (package, version, suite_name, architecture, reason, id)
else:
- Logger.log([reason, package, version, suite_name, architecture])
+ Logger.log([reason, package, version, suite_name, \
+ architecture, id])
idList.append(id)
return idList

--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master 1/3] replace msfl by dominate

2009-11-18T14:03:44Z

---
config/debian/cron.dinstall | 4 ++--
1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/debian/cron.dinstall b/config/debian/cron.dinstall
index e3c6f4e..747db2f 100755
--- a/config/debian/cron.dinstall
+++ b/config/debian/cron.dinstall
@@ -244,8 +244,8 @@ rm -f "$LOCK_ACCEPTED"
rm -f "$LOCK_NEW"

GO=(
- FUNC="msfl"
- TIME="make-suite-file-list"
+ FUNC="dominate"
+ TIME="dominate"
ARGS=""
ERR=""
)
--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...

[dak/master] add dominate.py and db upgrade script #25

2009-11-17T12:37:53Z

http://ftp-master.debian.org/wiki/projects/arch-all-tracking/
---
dak/dak.py | 4 +-
dak/dakdb/update25.py | 189 +++++++++++++++++++++++++++++++++++++++++++++++++
dak/dominate.py | 151 +++++++++++++++++++++++++++++++++++++++
dak/update_db.py | 2 +-
4 files changed, 344 insertions(+), 2 deletions(-)
create mode 100755 dak/dakdb/update25.py
create mode 100755 dak/dominate.py

diff --git a/dak/dak.py b/dak/dak.py
index 1d9336d..443e458 100755
--- a/dak/dak.py
+++ b/dak/dak.py
@@ -70,8 +70,10 @@ def init():
("process-upload",
"Process packages in queue/unchecked"),

+ ("dominate",
+ "Remove obsolete source and binary associations from suites"),
("make-suite-file-list",
- "Generate lists of packages per suite for apt-ftparchive"),
+ "OBSOLETE: replaced by dominate and generate-filelist"),
("make-pkg-file-mapping",
"Generate package <-> file mapping"),
("generate-filelist",
diff --git a/dak/dakdb/update25.py b/dak/dakdb/update25.py
new file mode 100755
index 0000000..b2813d9
--- /dev/null
+++ b/dak/dakdb/update25.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python
+
+"""
+Add views for new dominate command.
+
+@contact: Debian FTP Master <ftpmaster@...>
+@copyright: 2009 Torsten Werner <twerner@...>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import psycopg2
+
+def do_update(self):
+ print "Add views for generate_filelist to database."
+
+ try:
+ c = self.db.cursor()
+
+ print "Drop old views."
+ c.execute("DROP VIEW IF EXISTS binaries_suite_arch CASCADE")
+ c.execute("DROP VIEW IF EXISTS newest_all_associations CASCADE")
+ c.execute("DROP VIEW IF EXISTS obsolete_any_by_all_associations CASCADE")
+ c.execute("DROP VIEW IF EXISTS newest_any_associations CASCADE")
+ c.execute("DROP VIEW IF EXISTS obsolete_any_associations CASCADE")
+ c.execute("DROP VIEW IF EXISTS source_suite CASCADE")
+ c.execute("DROP VIEW IF EXISTS newest_source CASCADE")
+ c.execute("DROP VIEW IF EXISTS newest_src_association CASCADE")
+ c.execute("DROP VIEW IF EXISTS any_associations_source CASCADE")
+ c.execute("DROP VIEW IF EXISTS src_associations_src CASCADE")
+ c.execute("DROP VIEW IF EXISTS almost_obsolete_src_associations CASCADE")
+ c.execute("DROP VIEW IF EXISTS obsolete_src_associations CASCADE")
+ c.execute("DROP VIEW IF EXISTS bin_associations_binaries CASCADE")
+ c.execute("DROP VIEW IF EXISTS src_associations_bin CASCADE")
+ c.execute("DROP VIEW IF EXISTS almost_obsolete_all_associations CASCADE")
+ c.execute("DROP VIEW IF EXISTS obsolete_all_associations CASCADE")
+
+ print "Create new views."
+ c.execute("""
+CREATE VIEW binaries_suite_arch AS
+ SELECT bin_associations.id, binaries.id AS bin, binaries.package,
+ binaries.version, binaries.source, bin_associations.suite,
+ suite.suite_name, binaries.architecture, architecture.arch_string
+ FROM binaries JOIN bin_associations ON binaries.id = bin_associations.bin
+ JOIN suite ON suite.id = bin_associations.suite
+ JOIN architecture ON binaries.architecture = architecture.id;
+ """)
+ c.execute("""
+CREATE VIEW newest_all_associations AS
+ SELECT package, max(version) AS version, suite, architecture
+ FROM binaries_suite_arch
+ WHERE architecture = 2 GROUP BY package, suite, architecture;
+ """)
+ c.execute("""
+CREATE VIEW obsolete_any_by_all_associations AS
+ SELECT binaries_suite_arch.id, binaries_suite_arch.package,
+ binaries_suite_arch.version, binaries_suite_arch.suite,
+ binaries_suite_arch.architecture
+ FROM binaries_suite_arch
+ JOIN newest_all_associations
+ ON (binaries_suite_arch.package = newest_all_associations.package AND
+ binaries_suite_arch.version < newest_all_associations.version AND
+ binaries_suite_arch.suite = newest_all_associations.suite AND
+ binaries_suite_arch.architecture > 2);
+ """)
+ c.execute("""
+CREATE VIEW newest_any_associations AS
+ SELECT package, max(version) AS version, suite, architecture
+ FROM binaries_suite_arch
+ WHERE architecture > 2 GROUP BY package, suite, architecture;
+ """)
+ c.execute("""
+CREATE VIEW obsolete_any_associations AS
+ SELECT id, binaries_suite_arch.architecture, binaries_suite_arch.version,
+ binaries_suite_arch.package, binaries_suite_arch.suite
+ FROM binaries_suite_arch
+ JOIN newest_any_associations
+ ON binaries_suite_arch.architecture = newest_any_associations.architecture AND
+ binaries_suite_arch.package = newest_any_associations.package AND
+ binaries_suite_arch.suite = newest_any_associations.suite AND
+ binaries_suite_arch.version != newest_any_associations.version;
+ """)
+ c.execute("""
+CREATE VIEW source_suite AS
+ SELECT src_associations.id, source.id AS src , source.source, source.version,
+ src_associations.suite, suite.suite_name
+ FROM source
+ JOIN src_associations ON source.id = src_associations.source
+ JOIN suite ON suite.id = src_associations.suite;
+ """)
+ c.execute("""
+CREATE VIEW newest_source AS
+ SELECT source, max(version) AS version, suite
+ FROM source_suite
+ GROUP BY source, suite;
+ """)
+ c.execute("""
+CREATE VIEW newest_src_association AS
+ SELECT id, src, source, version, suite
+ FROM source_suite
+ JOIN newest_source USING (source, version, suite);
+ """)
+ c.execute("""
+CREATE VIEW any_associations_source AS
+ SELECT bin_associations.id, bin_associations.suite, binaries.id AS bin,
+ binaries.package, binaries.version AS binver, binaries.architecture,
+ source.id AS src, source.source, source.version AS srcver
+ FROM bin_associations
+ JOIN binaries ON bin_associations.bin = binaries.id AND architecture != 2
+ JOIN source ON binaries.source = source.id;
+ """)
+ c.execute("""
+CREATE VIEW src_associations_src AS
+ SELECT src_associations.id, src_associations.suite, source.id AS src,
+ source.source, source.version
+ FROM src_associations
+ JOIN source ON src_associations.source = source.id;
+ """)
+ c.execute("""
+CREATE VIEW almost_obsolete_src_associations AS
+ SELECT src_associations_src.id, src_associations_src.src,
+ src_associations_src.source, src_associations_src.version, suite
+ FROM src_associations_src
+ LEFT JOIN any_associations_source USING (src, suite)
+ WHERE bin IS NULL;
+ """)
+ c.execute("""
+CREATE VIEW obsolete_src_associations AS
+ SELECT almost.id, almost.src, almost.source, almost.version, almost.suite
+ FROM almost_obsolete_src_associations as almost
+ JOIN newest_src_association AS newest
+ ON almost.source = newest.source AND
+ almost.version < newest.version AND
+ almost.suite = newest.suite;
+ """)
+ c.execute("""
+CREATE VIEW bin_associations_binaries AS
+ SELECT bin_associations.id, bin_associations.bin, binaries.package,
+ binaries.version, bin_associations.suite, binaries.architecture
+ FROM bin_associations
+ JOIN binaries ON bin_associations.bin = binaries.id;
+ """)
+ c.execute("""
+CREATE VIEW src_associations_bin AS
+ SELECT src_associations.id, src_associations.source, src_associations.suite,
+ binaries.id AS bin, binaries.architecture
+ FROM src_associations
+ JOIN source ON src_associations.source = source.id
+ JOIN binaries ON source.id = binaries.source;
+ """)
+ c.execute("""
+CREATE VIEW almost_obsolete_all_associations AS
+ SELECT bin_associations_binaries.id AS id, bin, bin_associations_binaries.package,
+ bin_associations_binaries.version, suite
+ FROM bin_associations_binaries
+ LEFT JOIN src_associations_bin USING (bin, suite, architecture)
+ WHERE source IS NULL AND architecture = 2;
+ """)
+ c.execute("""
+CREATE VIEW obsolete_all_associations AS
+ SELECT almost.id, almost.bin, almost.package, almost.version, almost.suite
+ FROM almost_obsolete_all_associations AS almost
+ JOIN newest_all_associations AS newest
+ ON almost.package = newest.package AND
+ almost.version < newest.version AND
+ almost.suite = newest.suite;
+ """)
+
+ print "Committing"
+ c.execute("UPDATE config SET value = '25' WHERE name = 'db_revision'")
+ self.db.commit()
+
+ except psycopg2.InternalError, msg:
+ self.db.rollback()
+ raise DBUpdateError, "Database error, rollback issued. Error message : %s" % (str(msg))
+
diff --git a/dak/dominate.py b/dak/dominate.py
new file mode 100755
index 0000000..73eee8c
--- /dev/null
+++ b/dak/dominate.py
@@ -0,0 +1,151 @@
+#!/usr/bin/python
+
+"""
+Remove obsolete source and binary associations from suites.
+
+@contact: Debian FTP Master <ftpmaster@...>
+@copyright: 2009 Torsten Werner <twerner@...>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+from daklib.dbconn import *
+from daklib.config import Config
+from daklib import daklog, utils
+import apt_pkg, sys
+
+Options = None
+Logger = None
+
+def fetch(reason, query, args, session):
+ idList = []
+ for row in session.execute(query, args).fetchall():
+ (id, package, version, suite_name, architecture) = row
+ if Options['No-Action']:
+ print "Delete %s %s from %s architecture %s (%s)" % \
+ (package, version, suite_name, architecture, reason)
+ else:
+ Logger.log([reason, package, version, suite_name, architecture])
+ idList.append(id)
+ return idList
+
+def obsoleteAnyByAllAssociations(suite, session):
+ query = """
+ SELECT obsolete.id, package, obsolete.version, suite_name, arch_string
+ FROM obsolete_any_by_all_associations AS obsolete
+ JOIN architecture ON obsolete.architecture = architecture.id
+ JOIN suite ON obsolete.suite = suite.id
+ WHERE suite = :suite
+ """
+ return fetch('newer_all', query, { 'suite': suite }, session)
+
+def obsoleteAnyAssociations(suite, session):
+ query = """
+ SELECT obsolete.id, package, obsolete.version, suite_name, arch_string
+ FROM obsolete_any_associations AS obsolete
+ JOIN architecture ON obsolete.architecture = architecture.id
+ JOIN suite ON obsolete.suite = suite.id
+ WHERE suite = :suite
+ """
+ return fetch('newer_any', query, { 'suite': suite }, session)
+
+def obsoleteSrcAssociations(suite, session):
+ query = """
+ SELECT obsolete.id, source, obsolete.version, suite_name,
+ 'source' AS arch_string
+ FROM obsolete_src_associations AS obsolete
+ JOIN suite ON obsolete.suite = suite.id
+ WHERE suite = :suite
+ """
+ return fetch('old_and_unreferenced', query, { 'suite': suite }, session)
+
+def obsoleteAllAssociations(suite, session):
+ query = """
+ SELECT obsolete.id, package, obsolete.version, suite_name,
+ 'all' AS arch_string
+ FROM obsolete_all_associations AS obsolete
+ JOIN suite ON obsolete.suite = suite.id
+ WHERE suite = :suite
+ """
+ return fetch('old_and_unreferenced', query, { 'suite': suite }, session)
+
+def deleteAssociations(table, idList, session):
+ query = """
+ DELETE
+ FROM %s
+ WHERE id = :id
+ """ % table
+ session.execute(query, [{'id': id} for id in idList])
+
+def doDaDoDa(suite, session):
+ # keep this part disabled because it is too dangerous
+ #idList = obsoleteAnyByAllAssociations(suite, session)
+ #deleteAssociations('bin_associations', idList, session)
+
+ idList = obsoleteAnyAssociations(suite, session)
+ deleteAssociations('bin_associations', idList, session)
+
+ idList = obsoleteSrcAssociations(suite, session)
+ deleteAssociations('src_associations', idList, session)
+
+ idList = obsoleteAllAssociations(suite, session)
+ deleteAssociations('bin_associations', idList, session)
+
+def usage():
+ print """Usage: dak dominate [OPTIONS]
+Remove obsolete source and binary associations from suites.
+
+ -s, --suite=SUITE act on this suite
+ -h, --help show this help and exit
+ -n, --no-action don't commit changes
+ -f, --force also clean up untouchable suites
+
+SUITE can be comma (or space) separated list, e.g.
+ --suite=testing,unstable"""
+ sys.exit()
+
+def main():
+ global Options, Logger
+ cnf = Config()
+ Arguments = [('h', "help", "Obsolete::Options::Help"),
+ ('s', "suite", "Obsolete::Options::Suite", "HasArg"),
+ ('n', "no-action", "Obsolete::Options::No-Action"),
+ ('f', "force", "Obsolete::Options::Force")]
+ query_suites = DBConn().session().query(Suite)
+ suites = [suite.suite_name for suite in query_suites.all()]
+ if not cnf.has_key('Obsolete::Options::Suite'):
+ cnf['Obsolete::Options::Suite'] = ','.join(suites)
+ cnf['Obsolete::Options::Help'] = ''
+ cnf['Obsolete::Options::No-Action'] = ''
+ cnf['Obsolete::Options::Force'] = ''
+ apt_pkg.ParseCommandLine(cnf.Cnf, Arguments, sys.argv)
+ Options = cnf.SubTree("Obsolete::Options")
+ if Options['Help']:
+ usage()
+ Logger = daklog.Logger(cnf.Cnf, "dominate")
+ session = DBConn().session()
+ for suite_name in utils.split_args(Options['Suite']):
+ suite = session.query(Suite).filter_by(suite_name = suite_name).one()
+ if not suite.untouchable or Options['Force']:
+ doDaDoDa(suite.suite_id, session)
+ if Options['No-Action']:
+ session.rollback()
+ else:
+ session.commit()
+
+if __name__ == '__main__':
+ main()
+
diff --git a/dak/update_db.py b/dak/update_db.py
index ecdd99a..50b293f 100755
--- a/dak/update_db.py
+++ b/dak/update_db.py
@@ -45,7 +45,7 @@ from daklib.dak_exceptions import DBUpdateError
################################################################################

Cnf = None
-required_database_schema = 24
+required_database_schema = 25

################################################################################

--
1.6.3.3

--
To UNSUBSCRIBE, email to debian-dak-request@...
with a subject of "unsubscribe". Trouble? Contact listmaster@...