Defining a package to read JSON documents of unknown structure

(!******************************************************
   Mosel Example Programs
   ======================

   file json.mos
   `````````````
   Package providing advanced union operators for 
   representing a JSON document

   (c) 2022 Fair Isaac Corporation
       author: Y. Colombani, Nov. 2021
*******************************************************!)
package json
version 0.1.0
uses 'mmxml','mmsystem','mmhttp'

! **** Package type definitions ****
public declarations
 !@doc.descr Type definition: JSON object
  jobj=array(string) of any
 !@doc.descr Type definition: JSON array
  jarr=array(range) of any
 !@doc.descr Type definition: a null value is stored as a string
  jnull=string
 !@doc.descr Type definition: JSON value
 !@doc.info A 'jval' is either a scalar of type real, text, boolean, or jobj, jarr or jnull.
  jval=text or real or boolean or jobj or jarr or jnull
end-declarations

! **** [private] Parser related data ****
!@doc.ignore
declarations
  afct:array(range) of any       ! jparser callbacks
  public jsctx=
    record
      l:list of jval             ! List of active elements
    end-record
  datafile="forjson"+newmuid     ! Unique identifier for temp. filename
  jstxt=newmuid                  ! Unique identifier for temp. output filename
end-declarations

! **** Public subroutines ****

(!@doc.
  @descr Retrieve the indexing set of a 'jobj' entity
  @param o JSON object
  @return indexing set (set of labels occurring in the JSON object)
!)
public function getfields(o:jobj):set of string
  returned:=o.index(1)
end-function

(!@doc.
  @descr Retrieve the indexing (range) set of a 'jarr' entity
  @param o JSON array
  @return index range set (position count of objects within the JSON array)
  @info Index numbering starts with the value 1.
!)
public function getrange(o:jarr):range
  returned:=o.index(1)
end-function

(!@doc.
  @descr Load a JSON document
  @param fname (extended) file name of a JSON document
  @param doc structure for storing the JSON contents 
  @return 0 if successful, 1 in case of parsing error.
  @info The entity 'doc' passed as argument is reset by this function.
!)
public function loadjson(fname:text,doc:jval):integer
  declarations
    ctx:jsctx
  end-declarations
  reset(doc)
  fopen(fname,F_INPUT)
  returned:=jsonparse(afct,ctx)  ! Invoke the JSON parsing
  fclose(F_INPUT)
  if ctx.l.size>0 then
    doc:=ctx.l(1)
  end-if
end-function

(!@doc.
  @descr Parse a text as a JSON document
  @param data text containing JSON data
  @param doc structure for storing the JSON contents 
  @return 0 if successful, 1 in case of parsing error.
  @info The entity 'doc' passed as argument is reset by this function.
  @related Invokes <fctRef>loadjson</fctRef>
!)
public function parsejson(data:text,doc:jval):integer
  publish(datafile,data)
  returned:=loadjson(text("text:")+datafile,doc)
  unpublish(datafile)
end-function

(!@doc.
  @descr Create a JSON representation in text form for a Mosel entity
  @param mosobj a Mosel entity
  @param flag optional format configuration (see documentation of 'jsonwrite')
!)
public function jsontext(mosobj:any):text
  publish(jstxt,returned)
  jsonwrite("text:"+jstxt,mosobj)
  unpublish(jstxt)
end-function

public function jsontext(mosobj:any, flag:integer):text
  publish(jstxt,returned)
  jsonwrite("text:"+jstxt,mosobj,flag)
  unpublish(jstxt)
end-function

!---------------------------Internal subroutines----------------------------

! jparser callback: open an object
function js_open_object(ctx:jsctx, name:text):integer
  if ctx.l.size>0 then
    with o=ctx.l(ctx.l.size) do
      if o is jobj then
        sname:=string(name)
        create(o.jobj(sname).jobj)
        ctx.l+=[o.jobj(sname).jobj]
      else   ! jarr
        create(o.jarr(o.jarr.size+1).jobj)
        ctx.l+=[o.jarr(o.jarr.size).jobj]
      end-if
    end-do
  else
    ctx.l+=[(jobj)]
  end-if
end-function

! jparser callback: close an object
function js_close_object(ctx:jsctx):integer
  if ctx.l.size>1 then
    cuttail(ctx.l,1)
  end-if
end-function

! jparser callback: open an array
function js_open_array(ctx:jsctx, name:text):integer
  if ctx.l.size>0 then
    with o=ctx.l(ctx.l.size) do
      if o is jobj then
        sname:=string(name)
        create(o.jobj(sname).jarr)
        ctx.l+=[o.jobj(sname).jarr]
      else   ! jarr
        create(o.jarr(o.jarr.size+1).jarr)
        ctx.l+=[o.jarr(o.jarr.size).jarr]
      end-if
    end-do
  else
    ctx.l+=[(jarr)]
  end-if
end-function

! jparser callback: close an array
function js_close_array(ctx:jsctx):integer
  if ctx.l.size>1 then
    cuttail(ctx.l,1)
  end-if
end-function

! jparser callback: a textual value
function js_text_val(ctx:jsctx, name:text, type:integer, val:text):integer
  if ctx.l.size>0 then
    with o=ctx.l(ctx.l.size) do
      if o is jobj then
        o.jobj(string(name)):=val
      else   ! jarr
        o.jarr(o.jarr.size+1):=val
      end-if
    end-do
  else
    ctx.l+=[val]
  end-if
end-function

! jparser callback: a numerical value
function js_num_val(ctx:jsctx, name:text, val:real):integer
  if ctx.l.size>0 then
    with o=ctx.l(ctx.l.size) do
      if o is jobj then
        o.jobj(string(name)):=val
      else   ! jarr
        o.jarr(o.jarr.size+1):=val
      end-if
    end-do
  else
    ctx.l+=[val]
  end-if
end-function

! jparser callback: a Boolean value
function js_bool_val(ctx:jsctx, name:text, val:boolean):integer
  if ctx.l.size>0 then
    with o=ctx.l(ctx.l.size) do
      if o is jobj then
        o.jobj(string(name)):=val
      else   ! jarr
        o.jarr(o.jarr.size+1):=val
      end-if
    end-do
  else
    ctx.l+=[val]
  end-if
end-function

! jparser callback: the 'null' value
function js_null_val(ctx:jsctx, name:text):integer 
  if ctx.l.size>0 then
    with o=ctx.l(ctx.l.size) do
      if o is jobj then
        o.jobj(string(name)):="null"
      else   ! jarr
        o.jarr(o.jarr.size+1):="null"
      end-if
    end-do
  else
    ctx.l+=["null"]
  end-if
end-function

! Module initialisation (definition of JSON parser callbacks)
 afct(JSON_FCT_OPEN_OBJ):=->js_open_object
 afct(JSON_FCT_CLOSE_OBJ):=->js_close_object
 afct(JSON_FCT_OPEN_ARR):=->js_open_array
 afct(JSON_FCT_CLOSE_ARR):=->js_close_array
 afct(JSON_FCT_TEXT):=->js_text_val
 afct(JSON_FCT_NUM):=->js_num_val
 afct(JSON_FCT_BOOL):=->js_bool_val
 afct(JSON_FCT_NULL):=->js_null_val

end-package

(!******************************************************
   Mosel Example Programs
   ======================

   file readjson.mos
   `````````````````
   Alternative JSON reading methods, working with
   * an xmldoc structure, or 
   * specific user-defined record structures, or
   * union-based types defined by the package json.mos 

   -- Note: The package 'json' (file json.mos)  
      needs to be compiled before executing this model --

   (c) 2022 Fair Isaac Corporation
       author: S.Heipcke, Mar 2022
*******************************************************!)
model "Read book examples from JSON"
 uses "mmxml", "mmsystem", "mmhttp"
 uses ":json.bim"
! Uncomment the following line in place of the previous if you see the 
! compilation error message "Package ':json.bim' not found"
!  uses "json"     

 FNAME:="bookexamplesl.json"

!**** Using mmxml functionality to read the JSON file into an xmldoc structure
 declarations
   exampleDB,newdoc: xmldoc
   dirs,expls,files: list of integer
 end-declarations

 writeln("****Reading JSON into xmldoc structure****")
 jsonload(exampleDB,FNAME)
 save(exampleDB,"")   ! Display in XML format to visualize xmldoc representation
 root:=getnode(exampleDB,"*")
 if root>0 and getattr(exampleDB,root,"jst")="arr" then
   getnodes(exampleDB, "jsv/jsv[@jst='obj']", dirs)
   forall(d in dirs) do
     writeln(getvalue(exampleDB,getnode(exampleDB,d,"directory")), " - ", 
       getvalue(exampleDB,getnode(exampleDB,d,"title")) )
     getnodes(exampleDB, d, "models/jsv[@jst='obj']", expls)
     forall(m in expls)
       if getnode(exampleDB,m,"modFile")>0 then 
         writeln(" "*10, getvalue(exampleDB,getnode(exampleDB,m,"modTitle")),
           ": ", getvalue(exampleDB,getnode(exampleDB,m,"modFile")) )
       else
         getnodes(exampleDB,m,"modFileL/jsv",files)
         writeln(" "*10, getvalue(exampleDB,getnode(exampleDB,m,"modTitle")),
           ": ", union(f in files) [getvalue(exampleDB,f)] )
       end-if
   end-do
   asproc(copynode(exampleDB, getnode(exampleDB,"jsv/jsv[1]/models/jsv[1]"), 
     newdoc,0,XML_FIRSTCHILD))
   write("First example: "); jsonsave(newdoc,"")
   asproc(copynode(exampleDB, getnode(exampleDB,"jsv/jsv[2]"), 
     newdoc,0,XML_FIRSTCHILD))
   write("Second directory: "); jsonsave(newdoc,"")
 end-if

!**** User type definitions of specific record structures for reading
!**** the JSON file via jsonread of mmhttp    
 public declarations
   expl=public record
     id,modTitle,modType,modFeatures: text
     modFileL,modDataL: list of text
     modFile,modData: text
     modRating:integer
   end-record
   chap=public record
     id,title,directory: text
     models: list of expl
   end-record 
   examples: list of chap
 end-declarations

 writeln("\n****Reading JSON into record structures via jsonread****")
 setparam("ioctrl",true)
 jsonread(FNAME,examples)
 setparam("ioctrl",false)

 if getparam("iostatus")=0 then
   forall(dir in examples) do
     writeln(dir.directory, " - ", dir.title)
     forall(j in dir.models)
       writeln(" "*10, j.modTitle,": ", 
         if(j.modFile<>"", j.modFile, text(j.modFileL)) )
   end-do
   writeln("First example: ", jsontext(examples(1).models(1)))
   writeln("Second directory: ", jsontext(examples(2), HTTP_SKIP_EMPTYCOL))
 end-if

!**** Reading JSON file into generic structures based on union types defined
!**** by the package 'json'
 declarations
  expldb: jval
 end-declarations

 writeln("\n****Reading JSON into union type structures****")
 if loadjson(FNAME,expldb)=0 and expldb is jarr then
   forall(i in expldb.jarr.range)
     with dir=expldb.jarr(i).jobj do
       writeln(dir("directory")," - ", dir("title"))
       with modexpl=dir("models").jarr do
         forall(j in modexpl.range)
	   with ex=modexpl(j).jobj do
             writeln(" "*10, ex("modTitle"),": ", 	      
               if(isdefined(ex("modFile")), text(ex("modFile")), text(ex("modFileL"))))
	   end-do
       end-do
     end-do
   writeln("First example: ", jsontext(expldb.jarr(1).jobj("models").jarr(1)))
   writeln("Second directory: ", jsontext(expldb.jarr(2)))
 end-if

end-model