function jdata=jdataencode(data, varargin)
%
% jdata=jdataencode(data)
% or
% jdata=jdataencode(data, options)
% jdata=jdataencode(data, 'Param1',value1, 'Param2',value2,...)
%
% Annotate a MATLAB struct or cell array into a JData-compliant data
% structure as defined in the JData spec: http://github.com/fangq/jdata.
% This encoded form servers as an intermediate format that allows unambiguous
% storage, exchange of complex data structures and easy-to-serialize by
% json encoders such as savejson and jsonencode (MATLAB R2016b or newer)
%
% This function implements the JData Specification Draft 3 (Jun. 2020)
% see http://github.com/fangq/jdata for details
%
% author: Qianqian Fang (q.fang <at> neu.edu)
%
% input:
% data: a structure (array) or cell (array) to be encoded.
% options: (optional) a struct or Param/value pairs for user
% specified options (first in [.|.] is the default)
% AnnotateArray: [0|1] - if set to 1, convert all 1D/2D matrices
% to the annotated JData array format to preserve data types;
% N-D (N>2), complex and sparse arrays are encoded using the
% annotated format by default. Please set this option to 1 if
% you intend to use MATLAB's jsonencode to convert to JSON.
% Base64: [0|1] if set to 1, _ArrayZipData_ is assumed to
% be encoded with base64 format and need to be
% decoded first. This is needed for JSON but not
% UBJSON data
% Prefix: ['x0x5F'|'x'] for JData files loaded via loadjson/loadubjson, the
% default JData keyword prefix is 'x0x5F'; if the
% json file is loaded using matlab2018's
% jsondecode(), the prefix is 'x'; this function
% attempts to automatically determine the prefix;
% for octave, the default value is an empty string ''.
% UseArrayZipSize: [1|0] if set to 1, _ArrayZipSize_ will be added to
% store the "pre-processed" data dimensions, i.e.
% the original data stored in _ArrayData_, and then flaten
% _ArrayData_ into a row vector using row-major
% order; if set to 0, a 2D _ArrayData_ will be used
% UseArrayShape: [0|1] if set to 1, a matrix will be tested by
% to determine if it is diagonal, triangular, banded or
% toeplitz, and use _ArrayShape_ to encode the matrix
% MapAsStruct: [0|1] if set to 1, convert containers.Map into
% struct; otherwise, keep it as map
% Compression: ['zlib'|'gzip','lzma','lz4','lz4hc'] - use zlib method
% to compress data array
% CompressArraySize: [100|int]: only to compress an array if the
% total element count is larger than this number.
% FormatVersion [2|float]: set the JSONLab output version; since
% v2.0, JSONLab uses JData specification Draft 1
% for output format, it is incompatible with all
% previous releases; if old output is desired,
% please set FormatVersion to 1.9 or earlier.
%
% example:
% jd=jdataencode(struct('a',rand(5)+1i*rand(5),'b',[],'c',sparse(5,5)))
%
% encodedmat=jdataencode(single(magic(5)),'annotatearray',1,'prefix','x')
% jdatadecode(jsondecode(jsonencode(encodedmat))) % serialize by jsonencode
% jdatadecode(loadjson(savejson('',encodedmat))) % serialize by savejson
%
% encodedtoeplitz=jdataencode(uint8(toeplitz([1,2,3,4],[1,5,6])),'usearrayshape',1,'prefix','x')
% jdatadecode(jsondecode(jsonencode(encodedtoeplitz))) % serialize by jsonencode
% jdatadecode(loadjson(savejson('',encodedtoeplitz))) % serialize by savejson
%
% license:
% BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details
%
% -- this function is part of JSONLab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
%
if(nargin==0)
help jdataencode
return;
end
opt=varargin2struct(varargin{:});
if(isoctavemesh)
opt.prefix=jsonopt('Prefix','',opt);
else
opt.prefix=jsonopt('Prefix',sprintf('x0x%X','_'+0),opt);
end
opt.compression=jsonopt('Compression','',opt);
opt.nestarray=jsonopt('NestArray',0,opt);
opt.formatversion=jsonopt('FormatVersion',2,opt);
opt.compressarraysize=jsonopt('CompressArraySize',100,opt);
opt.base64=jsonopt('Base64',0,opt);
opt.mapasstruct=jsonopt('MapAsStruct',0,opt);
opt.usearrayzipsize=jsonopt('UseArrayZipSize',1,opt);
opt.messagepack=jsonopt('MessagePack',0,opt);
opt.usearrayshape=jsonopt('UseArrayShape',0,opt) && exist('bandwidth');
opt.annotatearray=jsonopt('AnnotateArray',0,opt);
jdata=obj2jd(data,opt);
%%-------------------------------------------------------------------------
function newitem=obj2jd(item,varargin)
if(iscell(item))
newitem=cell2jd(item,varargin{:});
elseif(isstruct(item))
newitem=struct2jd(item,varargin{:});
elseif(isnumeric(item) || islogical(item))
newitem=mat2jd(item,varargin{:});
elseif(ischar(item) || isa(item,'string'))
newitem=mat2jd(item,varargin{:});
elseif(isa(item,'containers.Map'))
newitem=map2jd(item,varargin{:});
elseif(isa(item,'categorical'))
newitem=cell2jd(cellstr(item),varargin{:});
elseif(isa(item,'function_handle'))
newitem=struct2jd(functions(item),varargin{:});
elseif(isa(item,'table'))
newitem=table2jd(item,varargin{:});
elseif(isa(item,'digraph') || isa(item,'graph'))
newitem=graph2jd(item,varargin{:});
elseif(isobject(item))
newitem=matlabobject2jd(item,varargin{:});
else
newitem=item;
end
%%-------------------------------------------------------------------------
function newitem=cell2jd(item,varargin)
newitem=cellfun(@(x) obj2jd(x, varargin{:}), item, 'UniformOutput',false);
%%-------------------------------------------------------------------------
function newitem=struct2jd(item,varargin)
num=numel(item);
if(num>1) % struct array
newitem=obj2jd(num2cell(item),varargin{:});
try
newitem=cell2mat(newitem);
catch
end
else % a single struct
names=fieldnames(item);
newitem=struct;
for i=1:length(names)
newitem.(names{i})=obj2jd(item.(names{i}),varargin{:});
end
end
%%-------------------------------------------------------------------------
function newitem=map2jd(item,varargin)
names=item.keys;
if(varargin{1}.mapasstruct) % convert a map to struct
newitem=struct;
if(~strcmp(item.KeyType,'char'))
data=num2cell(reshape([names, item.values],length(names),2),2);
for i=1:length(names)
data{i}{2}=obj2jd(data{i}{2},varargin{:});
end
newitem.(N_('_MapData_',varargin{:}))=data;
else
for i=1:length(names)
newitem.(N_(names{i},varargin{:}))=obj2jd(item(names{i}),varargin{:});
end
end
else % keep as a map and only encode its values
if(strcmp(item.KeyType,'char'))
newitem=containers.Map();
else
newitem=containers.Map('KeyType',item.KeyType,'ValueType','any');
end
for i=1:length(names)
newitem(names{i})=obj2jd(item(names{i}),varargin{:});
end
end
%%-------------------------------------------------------------------------
function newitem=mat2jd(item,varargin)
N=@(x) N_(x,varargin{:});
newitem=struct(N('_ArrayType_'),class(item),N('_ArraySize_'),size(item));
zipmethod=varargin{1}.compression;
minsize=varargin{1}.compressarraysize;
% 2d numerical (real/complex/sparse) arrays with _ArrayShape_ encoding enabled
if(varargin{1}.usearrayshape && ndims(item)==2 && ~isvector(item))
encoded=1;
if(~isreal(item))
newitem.(N('_ArrayIsComplex_'))=true;
end
symmtag='';
if(isreal(item) && issymmetric(double(item)))
symmtag='symm';
item=tril(item);
elseif(~isreal(item) && ishermitian(double(item)))
symmtag='herm';
item=tril(item);
end
[lband,uband]=bandwidth(double(item));
newitem.(N('_ArrayZipSize_'))=[lband+uband+1, min(size(item,1),size(item,2))];
if(lband+uband==0) % isdiag
newitem.(N('_ArrayShape_'))='diag';
newitem.(N('_ArrayData_'))=diag(item).';
elseif(uband==0 && lband==size(item,1)-1) % lower triangular
newitem.(N('_ArrayShape_'))=['lower' symmtag];
item=item.';
newitem.(N('_ArrayData_'))=item(triu(true(size(item)))).';
elseif(lband==0 && uband==size(item,2)-1) % upper triangular
newitem.(N('_ArrayShape_'))='upper';
item=item.';
newitem.(N('_ArrayData_'))=item(tril(true(size(item)))).';
elseif(lband==0) % upper band
newitem.(N('_ArrayShape_'))={'upperband',uband};
newitem.(N('_ArrayData_'))=spdiags(item.',-uband:lband).';
elseif(uband==0) % lower band
newitem.(N('_ArrayShape_'))={sprintf('lower%sband',symmtag),lband};
newitem.(N('_ArrayData_'))=spdiags(item.',-uband:lband).';
elseif(uband<size(item,2)-1 || lband<size(item,1)-1) % band
newitem.(N('_ArrayShape_'))={'band',uband,lband};
newitem.(N('_ArrayData_'))=spdiags(item.',-uband:lband).';
elseif(all(toeplitz(item(:,1),item(1,:))==item)) % Toeplitz matrix
newitem.(N('_ArrayShape_'))='toeplitz';
newitem.(N('_ArrayZipSize_'))=[2,max(size(item))];
newitem.(N('_ArrayData_'))=zeros(2,max(size(item)));
newitem.(N('_ArrayData_'))(1,1:size(item,2))=item(1,:);
newitem.(N('_ArrayData_'))(2,1:size(item,1))=item(:,1).';
else % full matrix
newitem=rmfield(newitem,N('_ArrayZipSize_'));
encoded=0;
end
% serialize complex data at last
if(encoded && isstruct(newitem) && ~isreal(newitem.(N('_ArrayData_'))))
item=squeeze(zeros([2, size(newitem.(N('_ArrayData_')))]));
item(1,:)=real(newitem.(N('_ArrayData_'))(:));
item(2,:)=imag(newitem.(N('_ArrayData_'))(:));
newitem.(N('_ArrayZipSize_'))=size(item);
newitem.(N('_ArrayData_'))=item;
end
% wrap _ArrayData_ into a single row vector, and store preprocessed
% size to _ArrayZipSize_ (force varargin{1}.usearrayzipsize=true)
if(encoded)
if(isstruct(newitem) && ~isvector(newitem.(N('_ArrayData_'))))
item=newitem.(N('_ArrayData_'));
item=permute(item,ndims(item):-1:1);
newitem.(N('_ArrayData_'))=item(:).';
else
newitem=rmfield(newitem,N('_ArrayZipSize_'));
end
newitem.(N('_ArrayData_'))=full(newitem.(N('_ArrayData_')));
return
end
end
% no encoding for char arrays or non-sparse real vectors
if(isempty(item) || isa(item,'string') || ischar(item) || varargin{1}.nestarray || ...
((isvector(item) || ndims(item)==2) && isreal(item) && ~issparse(item) && ...
~varargin{1}.annotatearray))
newitem=item;
return;
end
if(isa(item,'logical'))
item=uint8(item);
end
if(isreal(item))
if(issparse(item))
fulldata=full(item(item~=0));
newitem.(N('_ArrayIsSparse_'))=true;
newitem.(N('_ArrayZipSize_'))=[2+(~isvector(item)),length(fulldata)];
if(isvector(item))
newitem.(N('_ArrayData_'))=[find(item(:))', fulldata(:)'];
else
[ix,iy]=find(item);
newitem.(N('_ArrayData_'))=[ix(:)' , iy(:)', fulldata(:)'];
end
else
if(varargin{1}.formatversion>1.9)
item=permute(item,ndims(item):-1:1);
end
newitem.(N('_ArrayData_'))=item(:)';
end
else
newitem.(N('_ArrayIsComplex_'))=true;
if(issparse(item))
fulldata=full(item(item~=0));
newitem.(N('_ArrayIsSparse_'))=true;
newitem.(N('_ArrayZipSize_'))=[3+(~isvector(item)),length(fulldata)];
if(isvector(item))
newitem.(N('_ArrayData_'))=[find(item(:))', real(fulldata(:))', imag(fulldata(:))'];
else
[ix,iy]=find(item);
newitem.(N('_ArrayData_'))=[ix(:)' , iy(:)' , real(fulldata(:))', imag(fulldata(:))'];
end
else
if(varargin{1}.formatversion>1.9)
item=permute(item,ndims(item):-1:1);
end
newitem.(N('_ArrayZipSize_'))=[2,numel(item)];
newitem.(N('_ArrayData_'))=[real(item(:))', imag(item(:))'];
end
end
if(varargin{1}.usearrayzipsize==0 && isfield(newitem,N('_ArrayZipSize_')))
data=newitem.(N('_ArrayData_'));
data=reshape(data,fliplr(newitem.(N('_ArrayZipSize_'))));
newitem.(N('_ArrayData_'))=permute(data,ndims(data):-1:1);
newitem=rmfield(newitem,N('_ArrayZipSize_'));
end
if(~isempty(zipmethod) && numel(item)>minsize)
compfun=str2func([zipmethod 'encode']);
newitem.(N('_ArrayZipType_'))=lower(zipmethod);
if(~isfield(newitem,N('_ArrayZipSize_')))
newitem.(N('_ArrayZipSize_'))=size(newitem.(N('_ArrayData_')));
end
newitem.(N('_ArrayZipData_'))=compfun(typecast(newitem.(N('_ArrayData_'))(:).','uint8'));
newitem=rmfield(newitem,N('_ArrayData_'));
if(varargin{1}.base64)
newitem.(N('_ArrayZipData_'))=char(base64encode(newitem.(N('_ArrayZipData_'))));
end
end
if(isfield(newitem,N('_ArrayData_')) && isempty(newitem.(N('_ArrayData_'))))
newitem.(N('_ArrayData_'))=[];
end
%%-------------------------------------------------------------------------
function newitem=table2jd(item,varargin)
newitem=struct;
newitem.(N_('_TableCols_',varargin{:}))=item.Properties.VariableNames;
newitem.(N_('_TableRows_',varargin{:}))=item.Properties.RowNames';
newitem.(N_('_TableRecords_',varargin{:}))=table2cell(item);
%%-------------------------------------------------------------------------
function newitem=graph2jd(item,varargin)
newitem=struct;
nodedata=table2struct(item.Nodes);
if(isfield(nodedata,'Name'))
nodedata=rmfield(nodedata,'Name');
newitem.(N_('_GraphNodes_',varargin{:}))=containers.Map(item.Nodes.Name,num2cell(nodedata),'UniformValues',false);
else
newitem.(N_('_GraphNodes_',varargin{:}))=containers.Map(1:max(item.Edges.EndNodes(:)),num2cell(nodedata),'UniformValues',false);
end
edgenodes=num2cell(item.Edges.EndNodes);
edgedata=table2struct(item.Edges);
if(isfield(edgedata,'EndNodes'))
edgedata=rmfield(edgedata,'EndNodes');
end
edgenodes(:,3)=num2cell(edgedata);
if(isa(item,'graph'))
if(strcmp(varargin{1}.prefix,'x'))
newitem.(genvarname('_GraphEdges0_'))=edgenodes;
else
newitem.(encodevarname('_GraphEdges0_'))=edgenodes;
end
else
newitem.(N_('_GraphEdges_',varargin{:}))=edgenodes;
end
%%-------------------------------------------------------------------------
function newitem=matlabobject2jd(item,varargin)
try
if numel(item) == 0 %empty object
newitem = struct();
elseif numel(item) == 1 %
newitem = char(item);
else
propertynames = properties(item);
for p = 1:numel(propertynames)
for o = numel(item):-1:1 % aray of objects
newitem(o).(propertynames{p}) = item(o).(propertynames{p});
end
end
end
catch
newitem=any2jd(item,varargin{:});
end
%%-------------------------------------------------------------------------
function newitem=any2jd(item,varargin)
N=@(x) N_(x,varargin{:});
newitem.(N('_DataInfo_'))=struct('MATLABObjectClass',class(item),'MATLABObjectSize',size(item));
newitem.(N('_ByteStream_'))=getByteStreamFromArray(item); % use undocumented matlab function
if(varargin{1}.base64)
newitem.(N('_ByteStream_'))=char(base64encode(newitem.(N('_ByteStream_'))));
end
%%-------------------------------------------------------------------------
function newname=N_(name,varargin)
newname=[varargin{1}.prefix name];
