Support multiple bibliographies per topic (etc.)

The full list of changes is:
- Support multiple bibliographies per topic, i.e., in the
  'bibliography' metadata, allow topic entries to be list-valued
- Support the same topic structure for the 'references' metadata
  item, so can override references per topic
- Ignore duplicate references, i.e. if the same reference exists in
  multiple topics (the first one encountered is used, but this is not
  satisfactory because the topic processing order is indeterminate).
  Note that something had to be done here, because these duplicates
  would have the same ids and therefore would be ambiguous
- Take some changes from the diverged multiple-bibliographies repo,
  notably don't use utils.citeproc() because it doesn't have a 'quiet'
  option (which is needed for the second pass) [this is no longer fully
  the case; it's affected by some subsequent upstream changes]
- Fix an undefined 'orig_bib' bug (that probably caused no problems)
- Add filters to renumber citations in the case where they're referenced
  by citation number. These filters won't do anything in other cases,
  but nevertheless probably shouldn't be here

_extensions/multibib/multibib.lua

@@ -15,6 +15,8 @@ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 ]]
+
+-- --citeproc was added in 2.11, so we never use the old pandoc-citeproc
 PANDOC_VERSION:must_be_at_least '2.11'
 
 local pandoc = require 'pandoc'
@@ -32,16 +34,25 @@ local metatype = pandoc.utils.type or
 
 --- Collection of all cites in the document
 local all_cites = {}
+
 --- Document meta value
 local doc_meta = pandoc.Meta{}
 
 --- Div used by citeproc to insert the bibliography.
 local refs_div = pandoc.Div({}, pandoc.Attr('refs'))
 
+--- 'references' metadata for each topic
+local topic_refs = {}
+
 -- Div filled by citeproc with properties set according to
 -- the output format and the attributes of cs:bibliography
 local refs_div_with_properties
 
+-- Whether utils.citeproc() supports a 'quiet' argument
+-- (it doesn't yet, but perhaps it will, in which case this
+-- will use the appropriate pandoc version check)
+local supports_quiet_arg = false
+
 --- Run citeproc on a pandoc document
 --
 -- Falls back to the external `pandoc-citeproc` filter if the built-in
@@ -65,13 +76,28 @@ end
 --- Resolve citations in the document by combining all bibliographies
 -- before running citeproc on the full document.
 local function resolve_doc_citations (doc)
-  -- combine all bibliographies
+  -- combine all bibliographies and references
   local meta = doc.meta
   local bibconf = meta.bibliography
   meta.bibliography = pandoc.MetaList{}
   if metatype(bibconf) == 'table' then
     for _, value in pairs(bibconf) do
-      table.insert(meta.bibliography, stringify(value))
+      -- support list-valued items
+      if metatype(value) ~= 'List' then value = List{value} end
+      for _, val in ipairs(value) do
+        table.insert(meta.bibliography, stringify(val))
+      end
+    end
+  end
+  local refconf = meta.references
+  meta.references = pandoc.MetaList{}
+  if metatype(refconf) == 'table' then
+    for topic, refs in pairs(refconf) do
+      -- save topic references for meta_for_citeproc()
+      topic_refs[topic] = refs
+      for _, ref in ipairs(refs) do
+        table.insert(meta.references, ref)
+      end
     end
   end
   -- add refs div to catch the created bibliography
@@ -80,30 +106,53 @@ local function resolve_doc_citations (doc)
   doc = citeproc(doc)
   -- remove catch-all bibliography and keep it for future use
   refs_div_with_properties = table.remove(doc.blocks)
-  -- restore bibliography to original value
-  doc.meta.bibliography = orig_bib
+  -- restore bibliography and references to original values
+  doc.meta.bibliography = bibconf
+  doc.meta.references = refconf
   return doc
 end
 
---- Explicitly create a new meta object with all fields relevant for
---- pandoc-citeproc.
-local function meta_for_pandoc_citeproc (bibliography)
+--- Explicitly create a new meta object with all fields relevant for citeproc.
+local function meta_for_citeproc (bibliography, topic)
   -- We could just indiscriminately copy all meta fields, but let's be
   -- explicit about what's important.
   local fields = {
     'bibliography', 'references', 'csl', 'citation-style',
     'link-citations', 'citation-abbreviations', 'lang',
     'suppress-bibliography', 'reference-section-title',
-    'notes-after-punctuation', 'nocite'
+    'notes-after-punctuation', 'nocite', 'link-bibliography'
   }
   local new_meta = pandoc.Meta{}
   for _, field in ipairs(fields) do
-    new_meta[field] = doc_meta[field]
+    local value = doc_meta[field]
+    -- replace 'references' with the topic references
+    if field == 'references' and metatype(value) == 'table' and topic then
+      value = topic_refs[topic]
+    end
+    new_meta[field] = value
   end
   new_meta.bibliography = bibliography
   return new_meta
 end
 
+-- list of ref-xxx identifiers that have already been output
+local identifiers = List()
+
+-- ignore duplicate references (the first definition will win)
+local function ignore_duplicates(blocks)
+  local new_blocks = pandoc.Blocks{}
+  for _, block in ipairs(blocks) do
+    local identifier = block.attr.identifier
+    if not identifiers:includes(identifier) then
+      local new_block = pandoc.walk_block(block, {Span=_span})
+      new_blocks:insert(new_block)
+      identifiers:insert(identifier)
+    end
+  end
+
+  return new_blocks
+end
+
 local function remove_duplicates(classes)
   local seen = {}
   return classes:filter(function(x)
@@ -126,12 +175,12 @@ local function create_topic_bibliography (div)
     return nil
   end
   local tmp_blocks = {pandoc.Para(all_cites), refs_div}
-  local tmp_meta = meta_for_pandoc_citeproc(bibfile)
+  local tmp_meta = meta_for_citeproc(bibfile, name)
   local tmp_doc = pandoc.Pandoc(tmp_blocks, tmp_meta)
-  local res = citeproc(tmp_doc)
+  local res = citeproc(tmp_doc, true)
   -- First block of the result contains the dummy paragraph, second is
   -- the refs Div filled by citeproc.
-  div.content = res.blocks[2].content
+  div.content = ignore_duplicates(res.blocks[2].content)
   -- Set the classes and attributes as citeproc did it on refs_div
   div.classes = remove_duplicates(refs_div_with_properties.classes)
   div.attributes = refs_div_with_properties.attributes
@@ -145,6 +194,55 @@ local function create_topic_bibliography (div)
   return div
 end
 
+-- renumber numbered references and their citations
+-- (this logic should probably be in a separate filter; it's too
+-- dependent on the CSL, although it should do no harm)
+
+-- map from reference id to its new label
+local ref_map = List()
+
+-- ref counter
+local ref_counter = 1
+
+local function collect_numbered_refs(div)
+  if div.attr.classes:includes('csl-entry') then
+    local identifier = div.attr.identifier
+    local content = div.content
+    -- expect single Para with a Span (depending on style) possibly containing
+    -- the citation number (only do anything if it does)
+    if (#div.content > 0 and #div.content[1].content > 0 and
+        div.content[1].content[1].tag == 'Span') then
+      local span = div.content[1].content[1]
+      local content = span.content
+      if #content > 0 then
+        local text = content[1].text
+        local pre, num, post = content[1].text:match("^(%p*)(%d+)(%p*)$")
+        if pre and num and post then
+          local ident = identifier:gsub('^ref%-', '')
+          local label = string.format('%s%d%s', pre, ref_counter, post)
+          content[1] = pandoc.Str(label)
+          ref_map[ident] = label
+          ref_counter = ref_counter + 1
+          return div
+        end
+      end
+    end
+  end
+end
+
+local function renumber_cites(cite)
+  -- only consider cites with single citations
+  if #cite.citations == 1 then
+    local id = cite.citations[1].id
+    local label = ref_map[id]
+    -- only change the content if the label is defined
+    if label then
+      cite.content = label
+      return cite
+    end
+  end
+end
+
 return {
   {
     -- Collect all citations and the doc's Meta value for other filters.
@@ -153,4 +251,8 @@ return {
   },
   { Pandoc = resolve_doc_citations },
   { Div = create_topic_bibliography },
+
+  -- These should probably be handled via a separate filter.
+  { Div = collect_numbered_refs },
+  { Cite = renumber_cites }
 }