Editing Module:FormatTemplate/sandbox

---- This module is meant to produce a first pass translation from a template to a module.
---- It scans character by character, one time ONLY, using getletter().
---- It works by recognizing every beginning that the following features ''should'' not be intermingled: [[, {{, {{#, {{{
---- It works through which of these features is present by character-by-character scanning
---- As a result, a new stack[] table entry is created, according to how many levels deep in we are.
---- At each meaningful : or |, separating fields in these features, it accumulates existing text to stack[n].text
---- Within a field, stack.append() is used to attach text to stack[n].out (text to be quoted) or stack[n].code (accumulated with quotes, or statements)
---- For functions that are understood by it (stack.annotate) it translates the features to Lua code.
---- variables get their own little var[] array.  At the moment parameters are unique but parser functions may be reduplicated.
---- The beginnings and ends of features are labeled with colored spans, but most text is in nowiki tags.  
---- The result is generally any text to output to the new module with a need to wiki (br /'s) should go </nowiki> ... <nowiki>
---- The output should be postprocessed with another function, not yet written, to remove futile concatenations, lines marked for deletion, etc.
---- The output should be formatted with another function, not yet written, to impose proper indentation.

local p={}

-- constants
local MAXPOSN = 100000 -- usually 50000 was 3 seconds .. not right now though ..
local HOLDABLE = {["{"] = true, ["["] = true, ["}"] = true, ["]"] = true}
local ACTABLE = {["{"] = true, ["["] = true, ["}"] = true, ["]"] = true, ["|"] = true, [":"] = true}
local MARKER = {["{{{"] = "|", ["{{"] = "|", ["{{#"] = ":", ["[["] = "|"}
local MATCH = {["{{{"] = "}}}", ["{{#"] = "}}", ["{{"] = "}}", ["[["] = "]]"}
local ENDQUOTE = {['"'] = '"', ["'"] = "'", ["[=["] = "]=]", ["[==["] = "]==]", ["[===["] = "]===]", ["[====["] = "]====]", ["[=====["] = "]=====]", ["[======["] = "]======]", ["[=======["] = "]=======]", ["[========["] = "]========]", ["[=========["] = "]=========]", ["[==========["] = "]==========]", ["[===========["] = "]===========]", ["[============["] = "]============]", ["[=============["] = "]=============]"}
local DUPLICATECASES = 3 -- number of times to duplicate code setting a variable inside "then xxxx" cases rather than calculate it for all cases.

local text = ""
local debuglog = ""
local getletter -- this module is designed around reading ONCE, tracking state; getletter() gets each letter in text once
local flag = false -- true marks the end of the getletter() stream

function getContent(template)
    local title -- holds the result of the mw.title.xxx call
    if not(template) then
        title=mw.title.getCurrentTitle()
        if not(title) then return "error: failed to getCurrentTitle()" end
        local tdedoc=mw.ustring.match(title.fullText,"Template:(.-)/doc")
        if tdedoc then title=mw.title.new("Template:"..tdedoc) end -- SPECIAL CASE: Invoke in the template documentation processes the template instead
    else title=mw.title.new(page)
         if not (title) then return "error: failed to mw.title.new(" .. template .. ")" end
    end -- if not(template)
    return title.getContent(title) or ""
end

local function scanabort()
	flag = true
	return ":" -- an "actable" to prevent looping
end

local function escape(text)
    text = string.gsub(text,"%A%D","_")
    return text
end

local function quote(text)
	text = (not string.match(text,'"')) and ('"' .. text .. '"') or (not string.match(text,"'")) and ("'" .. text .. "'") or (not string.match(text,'%[=%[') and '[=[' .. text .. ']=]') or (not string.match(text,'%[==%[') and '[==[' .. text .. ']==]') or (not string.match(text,'%[===%[') and '[===[' .. text .. ']===]') or (not string.match(text,'%[====%[') and '[====[' .. text .. ']====]')
    return text
end

function translateModule(text,importstack,posn,template) -- note template is just for the error message
    local letter=""
    local output=""
    local outtable, vartable = {}, {}
    posn=tonumber(posn) or 0
    if posn>0 then text=string.sub(text,posn,-1) end --- need to chop off the preceding text so it doesn't gmatch to it
    local stopposn = (string.find(text, "[^{}%[%]|:]", MAXPOSN))
    if stopposn then text= string.sub(text, 1, stopposn) end

    -- vars holds all the parameters
    vars = {top = 0, list = {}}
    vars.declare = function(id, name)
    	if vars.list[name] then
    		return vars.list[name]
        else
    	    vars.list[name] = id
    	    return id
    	end
    end
    
    -- pars is like vars, for parser functions.  The entire stack entry has to be generated, and on pop, see if, minus its
    -- p_nnn name, it exactly matches a previous entry; if so, "drop" the old p_nnn to the last layer and add no new statement at all.
    -- this should work because otherwise nothing goes in and redefines p_nnn once it's set.
    pars = {list = {}}
    pars.declare = function(id, text)
    	text = string.gsub(text, id, "t_")
    	local pl = pars.list[text]
    	if pl then
    		return pl, false
    	else
    		pars.list[text] = id
    		return id, true
    	end
    end
    		

    -- stack is the main structure for holding the data.  Each time a feature is found you go up a level.
    -- When you go down you create a statement or append text into the stack a level lower (or both).
    stack = {top = #importstack, counter = 0}
    for i = 0, stack.top do
        stack[i] = {}
        stack[i].feature = importstack[i]
    	stack[i].text = {}
    	stack[i].seg = 1 -- this is NOT ACCURATE, would need to be saved in the transition
    end

    -- this table defines the code to deal with specific parser functions like {{#ifeq:  Also some functions for other features.
    stack.annotate = { -- name is the first field of the feature ('if' or the parameter name); id is the counter number
    	 -- text for the THESE STATEMENTS (stack top we're writing to)
    	 -- %1=if %2 then %3 else %4 end
    	 ['param1'] =  {function(last, text, varid) return '(' .. varid .. ' or ' .. '"{{{' .. string.sub(varid, 3, -1) .. '}}}")' end,
    	 	            etc = function(last, text) return " .. " .. quote("|" .. text) end, -- shouldn't happen......
    	 	            last = function(last, text) return text end},
    	 ['param2'] =  {function(last, text, varid) return "(" .. varid .. (last and "" or " or ") end,
    	 	            function(last, text) return quote(text) end, -- returned as _code_
    	 	            etc = function(last) return " .. " .. quote("|" .. text) end, -- shouldn't happen......
    	 	            last = function(last, text) return text .. ")" end},
    	 	            
    	 ['if'] =      {function(last) return "if mw.text.trim(" end, 
    		            function(last, text, varid)
    		            	if text == "" then 
    		            		text = '""'
    		            	end
    		            	text = text .. ') ~= "" then</nowiki><br><nowiki>&nbsp;&nbsp;&nbsp;&nbsp;'
    		            	if last then -- {{#if:%1}} will return empty string in any case.  This is a dumb way to do it, transitional...
    		            		text = text .. varid .. ' = ""</nowiki><br>else<br>&nbsp;&nbsp;&nbsp;&nbsp;<nowiki>' .. varid .. ' = ""'
    		            	else
    		            		text = text .. varid .. ' = '
    		            	end
    		            	return text
    		            end,
                        function(last, text, varid)
                        	if text == "" then
                        		text = '""'
                        	end
                        	if last then -- {{#if:%1|%2}} will return empty string if %1 is false
                        		text = text .. '</nowiki><br>else<br>&nbsp;&nbsp;&nbsp;&nbsp;<nowiki>' .. varid .. ' = ""'
                        	else 
                        		text = text .. '</nowiki><br>else<br>&nbsp;&nbsp;&nbsp;&nbsp;<nowiki>' .. varid .. ' = '
                        	end
                        	return text
                        end,
                        function(last, text, varid)
                        	if text == "" then -- this is %3 in {{#if:%1|%2|%3}}
                        		return '""'
                        	else
                        		return text
                        	end
                        end, -- this is stupid, but quotes are suppressed on null out string, so...
                        etc = function(last, text)
                        	return "</nowiki><br><nowiki>-- " .. text .. ': ignored' -- you can have {{#if:%1|%2|%3|%4}}, but %4 isn't displayed at all, even when %3 is
                        end,
                        last = function(last, text)
                        	return text .. "</nowiki><br><nowiki>end"
                        end}, -- weirdly, having function(last) there caused "Not enough memory" error, not nil truncation error
                        
         ['ifeq'] =    {function(last)
         	                return "if mw.text.trim("
         	            end, 
    		            function(last, text, varid)
    		            	return text .. ') == mw.text.trim('
    		            end,
    		            function(last, text, varid)
    		            	if text == "" then
    		            		text = '""'
    		            	end
    		            	text = text .. ') then</nowiki><br><nowiki>'
    		            	if last then
    		            		text = text .. varid .. ' = ""'
    		            	else
    		            		text = text .. varid .. ' = '
    		            	end
    		            	return text -- returned as _code_
    		            end,
                        function(last, text, varid)
                        	if text == "" then
                        		text = '""'
                        	end
                        	if last then
                        		text = text .. '</nowiki><br><nowiki>else ' .. varid .. ' = ""'
                        	else
                        		text = text .. '</nowiki><br><nowiki>else ' .. varid .. ' = '
                        	end
                        	return text
                        end,
                        function(last, text, varid)
                        	if text == "" then
                        		text = '""'
                        	end
                        	return text
                        end,
                        etc = function(last, text)
                        	return "</nowiki><br><nowiki>-- " .. text .. ": ignored"
                        end,
                        last = function(last, text)
                        	return text .. "</nowiki><br><nowiki>end"
                        end},
         ['switch'] =  {
         	            function(last, text, varid) return varid .. '_in = mw.text.trim(' end,
         	            function(last, text, varid) return text .. ")</nowiki><br><nowiki>if " .. varid .. "_in == mw.text.trim(" end,
         	            etc = function(last, text, varid)
                               -- debuglog = debuglog .. "text=" .. text
         	            	local text1 = string.match(text, "^([^=]*)")-- I don't know if I'll ever figure out what variable type string.match returns
         	            	local text2 = string.match(text, "=(.*)")
         	            	if text2 then
         	            	    local quotetype = string.sub(text1, 1, 1) -- text is received as a quoted string.  Need to split at the "=".
                                if (quotetype == "[") then quotetype = string.match(text1, "%[=*%[") end
                                text1 = text1 .. ENDQUOTE[quotetype]
                                text2 = quotetype .. text2
                            end
                                if last then
                                    if not text2 then
                                        return "'[TOKEN: DELETE LINE]') then</nowiki><br><nowiki>else " .. varid .. " = " .. text1 .. "</nowiki><br><nowiki>"
                                    elseif (string.match(text1, "#default")) then -- this isn't technically foolproof, but compared to the potential bugs in unquoting, whitespace, unicode, etc...
                                        return "'[TOKEN: DELETE LINE]') then</nowiki><br><nowiki>else " .. varid .. " = " .. text2 .. "</nowiki><br><nowiki>"
                                    else
                                       -- debuglog = debuglog  .. "DB" .. mw.text.trim(text1) .. "BD"
                                        return "'[TOKEN: DELETE LINE]') then</nowiki><br><nowiki>elseif " .. varid .. " == " .. text2 .. " then</nowiki><br><nowiki>&nbsp;&nbsp;&nbsp;&nbsp;" .. varid .. " = " .. text1 .. "</nowiki><br><nowiki>"
                                    end
                                else
         	            	        if not text2 then
         	            	            return text1 .. " or "
     	                	        else
                                        --  debuglog = debuglog .. "text1=" .. text1 .. "text2=" .. text2
     	            		            return text1 .. ") then</nowiki><br><nowiki>" .. varid .. " = mw.text.trim(" .. text2 .. ((not last) and (")</nowiki><br><nowiki>elseif " .. varid .. "_in == mw.text.trim(") or ")")
     	            		        end
                                end
     	                end,
     	                last = function(last, text) return text .. " end" end},
         	            
         ['parser'] = {
                        function (last, text, varid) return '"{{#" .. ' .. text end, -- '"</nowiki><span style="color:blue;">{{#</span><nowiki>" .. ' .. text end,
                        function(last, text, varid) return ' .. ":" .. ' .. text end, -- unknown parser functions are {{%1:%2|%3|etc.}}
                        etc = function(last, text, varid) return ' .. "|" .. ' .. text end,
                        last = function(last, text, varid) return text .. ' .. "}}"' end}, -- ' .. "</nowiki><span style="color:blue;">}}</span><nowiki>"' end},
         ['template'] = {
                        function (last, text, varid) return '"{{" .. ' .. text end, -- '"</nowiki><span style="color:red;">{{</span><nowiki>" .. ' .. text end,
                        function (last, text, varid)
                        	if varid == ':' then
                        		return ' .. "' .. varid .. '" .. ' .. text
                        	else
                        		return ' .. "|" .. ' .. text
                        	end
                        end,
                        etc = function(last, text, varid) return ' .. "|" .. ' .. text end,
                        last = function(last, text, varid) return text .. ' .. "}}"' end}, -- ' .. "</nowiki><span style="color:red;">}}</span><nowiki>"' end},
         ['wikilink'] = {
                        function (last, text, varid) return '"[[" .. ' .. text end, -- '"</nowiki><span style="color:green;">[[</span><nowiki>" .. ' .. text end,
                        etc = function(last, text, varid) return ' .. "|" .. ' .. text end,
                        last = function(last, text, varid) return text .. ' .. "]]"' end} } -- ' .. "</nowiki><span style="color:green;">]]</span><nowiki>"' end} }

    stack.append = function(text, codeflag) -- code is true if text doesn't need to be quoted
        -- stack[stack.top] shall contain two buffers of text, to which text can be added at the end by concatenation: .text, .code
        -- stack[stack.top].text is fully archived from previous fields
        -- stack[stack.top].code is ready to add to the Lua source, but is in the present field
        -- stack[stack.top].out is new text mode additions from the template.  It gets quoted and transferred to code whenever code is added.
        if text ~= "" then stack[stack.top].dropped = nil end
        if codeflag then
        	if (not stack[stack.top].code) then
        		stack[stack.top].code = text
        		stack[stack.top].start = true
        		return
        	end
            if stack[stack.top].out ~= "" then 
                if stack[stack.top].code ~= "" then stack[stack.top].code = stack[stack.top].code .. " .. " end
                if text ~= "" then text = " .. " .. text end
                stack[stack.top].code = stack[stack.top].code .. quote(stack[stack.top].out) .. text
            else
                stack[stack.top].code = stack[stack.top].code .. text
            end
            stack[stack.top].out = ""
        else
        	if (not stack[stack.top].out) then
        		stack[stack.top].out = text
        		return
        	end
            stack[stack.top].out = stack[stack.top].out .. text
        end
    end
    
    -- this is done at the end of a field (like at a "|" in a switch statement.
    -- fcn is some annotation function chosen per the stack's record of the name of the parser function for that stack level.
    stack.codify = function(fcn, id, name, last)
    	stack.append("", true)
    	stack[stack.top].dropped = nil
    	stack[stack.top].code = fcn(last, stack[stack.top].code, id, name) or stack[stack.top].code
    end
    
    -- this is done when a new feature is found; move up to the next level.
    -- Note stack[n] is a new table each time, so all its fields are nil by default.
    stack.push = function(feature)
        stack.top = stack.top + 1
        stack[stack.top] = {}
        if feature == "{{{" then
            stack[stack.top].text = {}
        else
            stack[stack.top].text = {"</nowiki><br /><nowiki>"}
        end
        stack[stack.top].seg = 1 -- which field we're in (IF 1 = 2 THEN 3 ELSE 4) delimited by | or :
        stack[stack.top].feature = feature -- actual text to open the sequence i.e. {{#
    end

    -- the end of a feature is found; now collapse its contents, or some memo, to the preceding level, or make a new statement about it.
    -- Note that in Wikitext the lower stack levels get input data _only_ from higher stack levels, and those are closed first, 
    -- so in theory a variable should always be set _before_ they're accessed.
    stack.pop = function(feature)
    	stack.field("", true)
        local spillover = ""
        local pop = stack[stack.top].feature
        if (MATCH[pop] ~= feature and feature == "}}}") then
            feature = "}}"
            spillover = "}"
        end
        if (MATCH[pop] ~= feature) then
            stack.append("<--- error? ")
        end
        stack.write()
        local feature = stack[stack.top].feature
        local name = stack[stack.top].name
        local id = stack[stack.top].id
        local text = table.concat(stack[stack.top].text)
        stack[stack.top] = nil
        stack.top = stack.top - 1
        if feature == "{{" or (feature == "{{#" and name == "parser") then -- wikitext
        	if stack[stack.top].dropped then text = " .. " .. text end
            stack.append(text, true) -- this is not lua code anymore here
            stack[stack.top].dropped = true
        elseif feature == "{{#" then -- parsed parser function, new line
        	local novel
        	id, novel = pars.declare(id, text)
        	if novel then table.insert(outtable, text) end
        	if stack[stack.top].dropped then id = " .. " .. id end
            stack.append(id, true)
            stack[stack.top].dropped = true
        elseif feature == "[[" then -- ordinary wikitext for display
        	if stack[stack.top].dropped then text = " .. " .. text end
        	stack.append(text, true) -- links are just quoted text, but for some reason "true" mode works and text mode doesn't....
        	stack[stack.top].dropped = true
        elseif feature == "{{{" then
        	if stack[stack.top].dropped then text = " .. " .. text end
        	stack.append(text, true) -- this is lua variable name for the parameter
        	stack[stack.top].dropped = true
        end
        return spillover
    end

    -- hit ":" or "|", the former used only in parser functions on the first field, or as a stand-in for the end of data
    stack.field = function (letter, last)
        local ss = stack[stack.top]
         -- first make sure this is really a field boundary
        if (last or (ss.seg == 1 and (letter == MARKER[ss.feature] or (ss.feature == "{{" and letter == ":"))) or (ss.feature ~= "[[" and ss.feature ~= "{{" and letter=="|")) then
             -- if we don't know, we need to know the name of this segment (unless we're just moving it through as text)
            if (ss.seg == 1) then
                local name = mw.text.trim(stack[stack.top].out) -- this misses a parser error if "#if :" is called.  Sue me.
                if (ss.feature == '{{#') then
        	        ss.name = stack.annotate[name] and name or "parser"
        	        ss.id = "p_" .. tostring(stack.counter)
        	        stack.counter = stack.counter + 1
                 -- now if the feature is {{{ then we register a variable name (
                elseif (ss.feature == '{{{') then
                    ss.name = name
                    ss.id = vars.declare("t_" .. escape(name), name) -- will be an old id if it exists
                    if last then ss.name = "param1" else ss.name = "param2" end
                elseif (ss.feature == '[[') then
                	ss.name = "wikilink"
                	ss.id = "_link_"
                else
                	ss.name = "template"
                    ss.id = letter -- i.e. in unimplemented function {{urlencode:, make the terminating character the id to distinguish
                end
              
            end
            stack.append("", true)
             -- now we know the name, I think.
            local fcn = stack.annotate[(not ss.id and "template") or ss.name]
            if fcn[ss.seg] then
            	stack.codify(fcn[ss.seg],ss.id, ss.name, last)
            else
                stack.codify(fcn["etc"],ss.id, ss.name, last) -- this is pretty muddy - using last both as a flag for preceding sections and as a function tack-on.  Should convert all uses of the second to the first.
            end
            if last then
            	stack.codify(fcn["last"],ss.id, ss.name)
            end
            ss.seg = ss.seg + 1
            stack.write()
        else -- NOT a new field, so don't do anything
            stack.append(letter)
        end
    end

    -- first uses append of "" as code to close out stack[n].code (this field's text contents)
    -- then inserts that in stack[n].text (simple concatenation only, stored until the stack is closed)
    stack.write = function()
        stack.append("", true) -- blank code addition forces quotation of remaining text
        table.insert(stack[stack.top].text, stack[stack.top].code)
        stack[stack.top].code = ""
    end

    -- this is the main loop to read character by character from the source text
    template=template or ""
    getletter = string.gmatch(text,".")
    stack[stack.top].out, stack[stack.top].code = "", ""
    repeat
        local holding = ""
        repeat
        	letter = letter or "" 
            while not ACTABLE[letter] do
    	        stack.append( letter)
    	        letter = getletter() or scanabort()
            end
            if HOLDABLE[letter] then
                holding = letter
            else
                stack.field(letter)
            end
            letter = ""
        until holding ~= "" or flag
        if #stack[stack.top].out>7500 then
            stack.write()
        end
        letter=getletter() or scanabort()
         -- add the letter to the next feature being parsed if possible
        if (holding == "[") then -- either [[ or just ignore
             -- cases based on the next letter after "["
            if (letter == "[") then
            	stack.push("[[")
                letter = ""
            else 
                stack.append( holding) -- single [, treat normally
            end
        elseif (holding == "{") then
             -- cases based on the next letter after "{"
            if (letter == "{") then
                letter = getletter() or scanabort()
               if (letter == "#") then
             	 stack.push("{{#")
                 letter = ""
               elseif (letter == "{") then
             	 stack.push("{{{")
             	 letter = ""
               else
             	 stack.push("{{")
               end
            end
        elseif (holding == "]") then
            if (letter == "]") then -- we have a ]]
                stack.pop("]]")
                letter = ""
            else stack.append(holding)
            end
        elseif (holding == "}") then
            if (letter == "}") then
                letter = getletter()
                if letter == "}" then
                    letter = stack.pop("}}}")
                else 
                    stack.pop("}}")
                end
            else stack.append(holding) -- lone } is nothing
            end
        end
    until flag
     -- a colon is used to indicate EOF to avoid checking for "flag" in the scan loop itself; this removes it
    stack[stack.top].out = string.sub(stack[stack.top].out, 1, -2)
    table.insert(vartable, "</nowiki>&nbsp;---- beginning of module ----<br />local p = {}<br /><br />getArgs = require('Module:Arguments').getArgs<br /><br />p.main = function(frame)<br />args = getArgs(frame)<br /><br /><nowiki>")
    table.insert(vartable, "</nowiki>&nbsp;-- begin variable table<br /><nowiki>")
    for k, v in pairs(vars.list) do
    	if (tonumber(k) == nil) or (tonumber(k) < 1) or (tonumber(k) % 1 ~= 0) then -- quote all non positive integers, others are numbers
            table.insert(vartable, "</nowiki>local " .. tostring(v) .. " = args['" .. k .. "'] and mw.text.trim(args['" .. k .. "'])<br /><nowiki>")
        else
        	table.insert(vartable, "</nowiki>local " .. tostring(v) .. " = args[" .. k .. "] and mw.text.trim(args[" .. k .. "])<br /><nowiki>")
        end
    end
    local pdeclare
    local pdeclareused
    local parssort = {}
    for k, v in pairs(pars.list) do
    	table.insert(parssort, tonumber(string.sub(v, 3, -1)))
    	pdeclareused = true
    end
    if parssort[1] then
    	table.sort(parssort)
    	pdeclare = "</nowiki>local p_" .. table.concat(parssort, ", p_") .. "<br /><nowiki>"
    else 
    	pdeclare = ""
    end
    table.insert(vartable, pdeclare)
    table.insert(vartable, "</nowiki>&nbsp;-- end variable table<br /><br />&nbsp;-- begin parser function translations<nowiki>")
    if stack.top>0 then
        stack[stack.top].out = string.sub(stack[stack.top].out, 1, -2) .. "<--- end of run ---><br /></nowiki>'''run incomplete.'''"
        stack.write()
        -- this code hasn't been updated - it doesn't contain all the information needed to resume a run!
        local stackcrypt = ""
        for i = stack.top, 1, -1 do
        	table.insert(outtable, table.concat(stack[i].text))
                stackcrypt = stackcrypt .. stack[i].feature
                stack[i] = {}
        end
        stack.top = 0
        stackcrypt=string.gsub(stackcrypt,"{","<")
        stackcrypt=string.gsub(stackcrypt,"%[","(")
        stackcrypt=string.gsub(stackcrypt,"}",">")
        stackcrypt=string.gsub(stackcrypt,"%]",")")
        if string.len(text) >= MAXPOSN then -- didn't complete the run, making false promises to the user now:
            stack[stack.top].out = stack[stack.top].out .. "<br />''Note: due to restrictions on Lua time usage, runs are truncated at MAXPOSN characters''"

            stack[stack.top].out = stack[stack.top].out .. "<br />''To continue this run, preview or enter <nowiki>{{#invoke:FormatTemplate|toModule|page="..template.."|stack="..stackcrypt.."|position="..#text.."}}"
        else stack[stack.top].out = stack[stack.top].out .. "<br />''If you have an additional segment of template to process, preview or enter <nowiki>{{#invoke:FormatTemplate|toModule|page="..template.."|stack="..stackcrypt.."|position=0}}"
        end
    end
    stack.write()
    -- var table is the variable declarations; outtable is the statements (if t_1 == "" then t_1 = "stuff"); stack[0] is the return statement
    output=table.concat(vartable) .. table.concat(outtable, "</nowiki><br /><nowiki>") .. "</nowiki><br />&nbsp;-- end parser function translations<br /><br /><nowiki>return frame:preprocess(" .. table.concat(stack[0].text, "</nowiki><br /><nowiki>") .. ")</nowiki><br />end<br />return p<br />&nbsp;---- end of module ----<br /><nowiki>"
    return output
end

function p._toModule(stackcrypt, posn)
	-- none of this stack/posn stuff actually works any more, it's tremendously out of date!
    stackcrypt=mw.ustring.gsub(stackcrypt,"<","{")
    stackcrypt=mw.ustring.gsub(stackcrypt,"%(","[")
    stackcrypt=mw.ustring.gsub(stackcrypt,">","}")
    stackcrypt=mw.ustring.gsub(stackcrypt,"%)","]")
    local stack={}
    local prowl=mw.ustring.gmatch(stackcrypt,"[^,%s]+")
    repeat
        local x=prowl()
        if x then table.insert(stack,x) end
    until not x
    local nowikisafehouse={}
    local nowikielementnumber=0
    local prowl=mw.ustring.gmatch(text,"(<nowiki>.-<%/nowiki>)")
    repeat
        local nowikimatch=prowl()
        if not(nowikimatch) then break end
        nowikimatch = mw.ustring.gsub(nowikimatch, "<", "&lt;") -- I want these inactive on display
        nowikielementnumber=nowikielementnumber+1
        table.insert(nowikisafehouse,nowikimatch)
    until false
    local nowikicount
    text, nowikicount = mw.ustring.gsub(text,"(<nowiki>.-<%/nowiki>)","<Module:FormatTemplate internal nowiki token>")
    debuglog = debuglog .. "-- " .. tostring(nowikicount) .. " nowiki segments treated as plain text"
    text = mw.ustring.gsub(text, "\n", '\\n') -- trying preserving ALL newlines
     -- this is the meat of the formatting
    text=translateModule(text,stack,posn,page)
     -- unprotect the nowikis from the template itself - but inactivate them on first display!
    for nw = 1,nowikielementnumber do
        text=mw.ustring.gsub(text,"<Module:FormatTemplate internal nowiki token>", nowikisafehouse[nw], 1)
    end
    text = "<nowiki>" .. text .. "</nowiki>"
    -- returns global changes in text, debuglog
end

function nextquote(posn)
	-- starting from a position that we know is not in a string, comment, etc.!
	local quotes = {['(")'] = function() return '"' end, 
	                ["(')"] = function() return "'" end, 
	                ["(%[=+%[)"] = function(q) return "]" .. string.sub(q, 2, -2) .. "]" end}
    local startquoteposn = #text
    local loc, xxx, startquotetype, endquotetype, endquoteposn
	for k,v in pairs(quotes) do
		loc, xxx, startquotetype = string.find(text, k, posn)
		if (loc and loc < startquoteposn) then
			startquoteposn = loc
			endquotetype = v(startquotetype)
		end
    end
    if (startquoteposn == #text) then return nil end -- no more quotes!
    xxx, endquoteposn = string.find(text, endquotetype, startquoteposn + 1)
    return startquoteposn, endquoteposn, endquotetype
end

function p._fixIfs()
	-- receives global 'text'
	-- if a parser variable will only be used within "a few" if statements, only calculate it there.
	-- (in theory this applies to params, but it's nicer to have a full table up front)
	text = string.gsub(text, "\n[^\n]-%[TOKEN: DELETE LINE%][^\n]-\n", "\n") -- finally clear up those delete line tokens.  Might move to earlier pass...
	local pdec = string.match(text, "local (p_.-)\n")
	assert(pdec, "failed to find 'local p_nnn...' variable list")
	pdec = mw.text.split(pdec, ", ") -- pdec is now a sequence of all the p_nnn variables in the order they are first mentioned in the first pass code
	local xxx, parssegstart, parssegend -- xxx = throwaway value.  This function only messes with the scope of parser function evaluations.
	xxx, parssegstart = string.find(text, "\n%s%-%- begin parser function translations\n")
	parssegend = string.find(text, "\n%s%-%- end parser function translations\n", parssegstart)
	debuglog = debuglog .. tostring(parssegstart) .. "FFF" .. tostring(parssegend)
	local parsseg = string.sub(text, parssegstart, parssegend)
	local parsarray = mw.text.split(parsseg, "\n\n")
	-- parsarray is now made up of individual popped stack entries from the first pass.  #if, #ifeq, #switch, for example.
	local parsusages = {} -- which variables appear in one parsseg (dictionary)
	local varusages = {} -- which parssegs a variable appears in (sequence)
	for i = 1, #pdec do
		varusages[pdec[i]] = {}
	end
	for i = 1, #parsarray do
		local getvar = string.gmatch(parsarray[i], "(p_%d*)")
		parsusages[i] = {}
		repeat
			var = getvar()
			if not var then break end
			debuglog = debuglog .. "-" .. var
			parsusages[i][var] = true
			table.insert(varusages[var], i)
		until false
	end
	-- Now we go through the variables last to first.  If a p_variable is used only in DUPLICATECASES or fewer "then XXX" statements, insert the code setting it after 'then'
	for i = #pdec, 1, -1 do
		local var = pdec[i]
		if #varusages[var] <= DUPLICATECASES then
			-- OK, now each variable ought to be assigned only the first time it is mentioned, used all the others
			-- first, for now we're not trying to do anything with variables called in the decision making logic (the if, or assignment of p_nnn_in
			local predicate = false
			for c = 2, #varusages[var] do
                local xxx, predend = string.find(parsarray[i], "then\n")
                if not predend or string.find(string.sub(parsarray[i], 1, predend), var) then
                	predicate = true
                -- now figure out where/which line to substitute, but don't do it until all pass the test
            end
            if not predicate then
            	-- substitute these segments, blank out the entries for the moved segment
            end
		end
	end
		
	end
	text = mw.text.nowiki(text) -- just to mark for now
    debuglog = "\nDebug data:\n" .. mw.text.nowiki(debuglog)
    -- returns global 'text', 'debuglog'
end
	
function p._fixConcats()
	-- receives global 'text'
	-- tokenize comments to avoid getting fouled
	local outarray = {}
	posn = 1
	quote1 = {nextquote(1)}
	repeat
		quote2 = {nextquote(quote1[2]+1)}
	    if not quote2[1] then break end
		-- do something with the concat if it's just a concat
		if (mw.text.trim(string.sub(text, quote1[2] + 1, quote2[1] - 1)) == "..") then
			if (quote1[3] == quote2[3]) then
				table.insert(outarray, string.sub(text, posn, quote1[2] - 1))
				posn = quote2[1] + 1
			end
	    end
		quote1 = quote2
	until false
	table.insert(outarray, string.sub(text, posn, -1))
	text = "<pre>" .. mw.text.nowiki(table.concat(outarray)) .. "</pre>"
end

-- this function also (mis)handles the pre-processing to get rid of nowikis and comments (only it doesn't do the comments yet, etc...) 
function p.main(frame,fcn)
    local args=frame.args
    local parent=frame.getParent(frame)
    if parent then pargs=parent.args else pargs={} end
    page=args.page or pargs.page
    text = getContent(page)
    local stackcrypt=args.stack or pargs.stack or ""
    local posn=args.position or pargs.position or 0
     -- decide on a function
    fcn=fcn or args["function"] or pargs["function"] or ""
    fcn=mw.ustring.match(fcn,"%S+")
    if (fcn == "toModule") then
    	p._toModule(stackcrypt, posn) -- passing text, debuglog globally
    elseif (fcn == "fixConcats") then
    	p._fixConcats()
    elseif (fcn == "fixIfs") then
    	p._fixIfs()
    end
     -- preprocess as nowiki-bounded text
    return frame:preprocess(text .. "\n" .. debuglog)
end

function p.toModule(frame)
    return p.main(frame,"toModule")
end

return p