Skip to content

Commit

Permalink
decode graph data, encode non-char-keyed map data
Browse files Browse the repository at this point in the history
  • Loading branch information
@fangq
fangq committed Oct 21, 2019
1 parent 11712b7 commit 874945f
Show file tree
Hide file tree
Showing 7 changed files with 227 additions and 68 deletions.
125 changes: 102 additions & 23 deletions jdatadecode.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,18 @@
% "_ArrayZipType_", "_ArrayZipSize", "_ArrayZipData_"
% opt: (optional) a list of 'Param',value pairs for additional options
% The supported options include
% 'Recursive', if set to 1, will apply the conversion to
% Recursive: [1|0] if set to 1, will apply the conversion to
% every child; 0 to disable
% 'Base64'. if set to 1, _ArrayZipData_ is assumed to
% Base64: [0|1] if set to 1, _ArrayZipData_ is assumed to
% be encoded with base64 format and need to be
% decoded first. This is needed for JSON but not
% UBJSON data
% 'Prefix', for JData files loaded via loadjson/loadubjson, the
% Prefix: ['x0x5F'|'x'] for JData files loaded via loadjson/loadubjson, the
% default JData keyword prefix is 'x0x5F'; if the
% json file is loaded using matlab2018's
% jsondecode(), the prefix is 'x'; this function
% attempts to automatically determine the prefix.
% 'FormatVersion' [2|float]: set the JSONLab output version;
% FormatVersion: [2|float]: set the JSONLab output version;
% since v2.0, JSONLab uses JData specification Draft 1
% for output format, it is incompatible with all
% previous releases; if old output is desired,
Expand All @@ -51,27 +51,38 @@
%

newdata=data;
opt=varargin2struct(varargin{:});

%% process non-structure inputs
if(~isstruct(data))
if(iscell(data))
newdata=cellfun(@(x) jdatadecode(x),data,'UniformOutput',false);
newdata=cellfun(@(x) jdatadecode(x,opt),data,'UniformOutput',false);
elseif(isa(data,'containers.Map'))
newdata=containers.Map('KeyType',data.KeyType,'ValueType','any');
names=data.keys;
for i=1:length(names)
newdata(names{i})=jdatadecode(data(names{i}),opt);
end
end
return;
end

%% assume the input is a struct below
fn=fieldnames(data);
len=length(data);
opt=varargin2struct(varargin{:});
needbase64=jsonopt('Base64',1,opt);
needbase64=jsonopt('Base64',0,opt);
format=jsonopt('FormatVersion',2,opt);
prefix=jsonopt('Prefix',sprintf('x0x%X','_'+0),opt);
if(isempty(strmatch(N_('_ArrayType_'),fn)) && ~isempty(strmatch('x_ArrayType_',fn)))
if(~any(ismember(N_('_ArrayType_'),fn)) && any(ismember('x_ArrayType_',fn)))
prefix='x';
opt.prefix='x';
end

%% recursively process subfields
if(jsonopt('Recursive',1,opt)==1)
for i=1:length(fn) % depth-first
for j=1:len
if(isstruct(data(j).(fn{i})))
if(isstruct(data(j).(fn{i})) || isa(data(j).(fn{i}),'containers.Map'))
newdata(j).(fn{i})=jdatadecode(data(j).(fn{i}),opt);
elseif(iscell(data(j).(fn{i})))
newdata(j).(fn{i})=cellfun(@(x) jdatadecode(x,opt),newdata(j).(fn{i}),'UniformOutput',false);
Expand All @@ -81,15 +92,15 @@
end

%% handle array data
if(~isempty(strmatch(N_('_ArrayType_'),fn)) && (~isempty(strmatch(N_('_ArrayData_'),fn)) || ~isempty(strmatch(N_('_ArrayZipData_'),fn))))
if(any(ismember(N_('_ArrayType_'),fn)) && (any(ismember(N_('_ArrayData_'),fn)) || any(ismember(N_('_ArrayZipData_'),fn))))
newdata=cell(len,1);
for j=1:len
if(~isempty(strmatch(N_('_ArrayZipSize_'),fn)) && ~isempty(strmatch(N_('_ArrayZipData_'),fn)))
if(any(ismember(N_('_ArrayZipSize_'),fn)) && any(ismember(N_('_ArrayZipData_'),fn)))
zipmethod='zip';
if(~isempty(strmatch(N_('_ArrayZipType_'),fn)))
if(any(ismember(N_('_ArrayZipType_'),fn)))
zipmethod=data(j).(N_('_ArrayZipType_'));
end
if(~isempty(strmatch(zipmethod,{'zlib','gzip','lzma','lzip','lz4','lz4hc'})))
if(any(ismember(zipmethod,{'zlib','gzip','lzma','lzip','lz4','lz4hc'})))
decompfun=str2func([zipmethod 'decode']);
if(needbase64)
ndata=reshape(typecast(decompfun(base64decode(data(j).(N_('_ArrayZipData_')))),data(j).(N_('_ArrayType_'))),data(j).(N_('_ArrayZipSize_'))(:)');
Expand All @@ -105,18 +116,18 @@
end
ndata=cast(data(j).(N_('_ArrayData_')),char(data(j).(N_('_ArrayType_'))));
end
if(~isempty(strmatch(N_('_ArrayZipSize_'),fn)))
if(any(ismember(N_('_ArrayZipSize_'),fn)))
ndata=reshape(ndata(:),fliplr(data(j).(N_('_ArrayZipSize_'))(:)'));
ndata=permute(ndata,ndims(ndata):-1:1);
end
iscpx=0;
if(~isempty(strmatch(N_('_ArrayIsComplex_'),fn)))
if(any(ismember(N_('_ArrayIsComplex_'),fn)))
if(data(j).(N_('_ArrayIsComplex_')))
iscpx=1;
end
end
if(~isempty(strmatch(N_('_ArrayIsSparse_'),fn)) && data(j).(N_('_ArrayIsSparse_')))
if(~isempty(strmatch(N_('_ArraySize_'),fn)))
if(any(ismember(N_('_ArrayIsSparse_'),fn)) && data(j).(N_('_ArrayIsSparse_')))
if(any(ismember(N_('_ArraySize_'),fn)))
dim=double(data(j).(N_('_ArraySize_'))(:)');
if(iscpx)
ndata(end-1,:)=complex(ndata(end-1,:),ndata(end,:));
Expand All @@ -140,7 +151,7 @@
end
ndata=sparse(ndata(1,:),ndata(2,:),ndata(3,:));
end
elseif(~isempty(strmatch(N_('_ArraySize_'),fn)))
elseif(any(ismember(N_('_ArraySize_'),fn)))
if(iscpx)
ndata=complex(ndata(1,:),ndata(2,:));
end
Expand All @@ -161,7 +172,7 @@
end

%% handle table data
if(~isempty(strmatch(N_('_TableRecords_'),fn)))
if(any(ismember(N_('_TableRecords_'),fn)))
newdata=cell(len,1);
for j=1:len
ndata=data(j).(N_('_TableRecords_'));
Expand Down Expand Up @@ -191,14 +202,14 @@
end

%% handle map data
if(~isempty(strmatch(N_('_MapData_'),fn)))
if(any(ismember(N_('_MapData_'),fn)))
newdata=cell(len,1);
for j=1:len
key={};
val={};
key=cell(1,length(data(j).(N_('_MapData_'))));
val=cell(size(key));
for k=1:length(data(j).(N_('_MapData_')))
key{k}=data(j).(N_('_MapData_')){k}{1};
val{k}=data(j).(N_('_MapData_')){k}{2};
val{k}=jdatadecode(data(j).(N_('_MapData_')){k}{2},opt);
end
ndata=containers.Map(key,val);
newdata{j}=ndata;
Expand All @@ -207,6 +218,74 @@
newdata=newdata{1};
end
end

%% handle graph data
if(any(ismember(N_('_GraphNodes_'),fn)) && exist('graph','file') && exist('digraph','file'))
newdata=cell(len,1);
isdirected=1;
for j=1:len
nodedata=data(j).(N_('_GraphNodes_'));
if(isstruct(nodedata))
nodetable=struct2table(nodedata);
elseif(isa(nodedata,'containers.Map'))
nodetable=[keys(nodedata);values(nodedata)];
if(strcmp(nodedata.KeyType,'char'))
nodetable=table(nodetable(1,:)',nodetable(2,:)','VariableNames',{'Name','Data'});
else
nodetable=table(nodetable(2,:)','VariableNames',{'Data'});
end
else
nodetable=table;
end

if(any(ismember(N_('_GraphEdges_'),fn)))
edgedata=data(j).(N_('_GraphEdges_'));
elseif(any(ismember(N_('_GraphEdges0_'),fn)))
edgedata=data(j).(N_('_GraphEdges0_'));
isdirected=0;
elseif(any(ismember(N_('_GraphMatrix_'),fn)))
edgedata=jdatadecode(data(j).(N_('_GraphMatrix_')),varargin{:});
end

if(exist('edgedata','var'))
if(iscell(edgedata))
endnodes=edgedata(:,1:2);
endnodes=reshape([endnodes{:}],size(edgedata,1),2);
weight=cell2mat(edgedata(:,3:end));
edgetable=table(endnodes,[weight.Weight]','VariableNames',{'EndNodes','Weight'});

if(isdirected)
newdata{j}=digraph(edgetable,nodetable);
else
newdata{j}=graph(edgetable,nodetable);
end
elseif(ismatrix(edgedata) && isstruct(nodetable))
newdata{j}=digraph(edgedata,fieldnames(nodetable));
end
end
end
if(len==1)
newdata=newdata{1};
end
end

%% handle bytestream and arbitrary matlab objects
if(sum(ismember({N_('_ByteStream_'),N_('_DataInfo_')},fn))==2)
newdata=cell(len,1);
for j=1:len
if(isfield(data(j).(N_('_DataInfo_')),'MATLABObjectClass'))
if(needbase64)
newdata{j}=getArrayFromByteStream(base64decode(data(j).(N_('_ByteStream_'))));
else
newdata{j}=getArrayFromByteStream(data(j).(N_('_ByteStream_')));
end
end
end
if(len==1)
newdata=newdata{1};
end
end

%% subfunctions
function escaped=N_(str)
escaped=[prefix str];
Expand Down
68 changes: 47 additions & 21 deletions jdataencode.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,6 @@
end

opt=varargin2struct(varargin{:});
opt.prefix=jsonopt('Prefix',sprintf('x0x%X','_'+0),opt);

jdata=obj2jd(data,opt);

%%-------------------------------------------------------------------------
Expand All @@ -61,6 +59,8 @@
newitem=cell2jd(item,varargin{:});
elseif(isstruct(item))
newitem=struct2jd(item,varargin{:});
elseif(isnumeric(item) || islogical(item))
newitem=mat2jd(item,varargin{:});
elseif(ischar(item) || isa(item,'string'))
newitem=mat2jd(item,varargin{:});
elseif(isa(item,'containers.Map'))
Expand All @@ -69,14 +69,14 @@
newitem=cell2jd(cellstr(item),varargin{:});
elseif(isa(item,'function_handle'))
newitem=struct2jd(functions(item),varargin{:});
elseif(islogical(item) || isnumeric(item))
newitem=mat2jd(item,varargin{:});
elseif(isa(item,'table'))
newitem=table2jd(item,varargin{:});
elseif(isa(item,'digraph') || isa(item,'graph'))
newitem=graph2jd(item,varargin{:});
else
elseif(~isoctavemesh)
newitem=any2jd(item,varargin{:});
else
newitem=item;
end

%%-------------------------------------------------------------------------
Expand All @@ -88,13 +88,13 @@
function newitem=struct2jd(item,varargin)

num=numel(item);
if(num>1)
if(num>1) % struct array
newitem=obj2jd(num2cell(item),varargin{:});
try
newitem=cell2mat(newitem);
catch
end
else
else % a single struct
names=fieldnames(item);
newitem=struct;
for i=1:length(names)
Expand All @@ -106,19 +106,28 @@
function newitem=map2jd(item,varargin)

names=item.keys;
if(jsonopt('MapAsStruct',0,varargin{:}))
if(jsonopt('MapAsStruct',0,varargin{:})) % convert a map to struct
newitem=struct;
for i=1:length(names)
newitem(N_(names{i},varargin{:}))=obj2jd(item(names{i}),varargin{:});
if(~strcmp(item.KeyType,'char'))
data=num2cell(reshape([names, item.values],length(names),2),2);
for i=1:length(names)
data{i}{2}=obj2jd(data{i}{2},varargin{:});
end
newitem.(N_('_MapData_',varargin{:}))=data;
else
for i=1:length(names)
newitem.(N_(names{i},varargin{:}))=obj2jd(item(names{i}),varargin{:});
end
end
else
newitem=containers.Map;
else % keep as a map and only encode its values
newitem=containers.Map('KeyType',item.KeyType,'ValueType','any');
for i=1:length(names)
newitem(names{i})=obj2jd(item(names{i}),varargin{:});
end
end
%%-------------------------------------------------------------------------
function newitem=mat2jd(item,varargin)

if(isempty(item) || isa(item,'string') || ischar(item) || (isvector(item) && isreal(item) && ~issparse(item)))
newitem=item;
return;
Expand Down Expand Up @@ -173,6 +182,7 @@
data=reshape(data,fliplr(newitem.(N('_ArrayZipSize_'))));
newitem.(N('_ArrayData_'))=permute(data,ndims(data):-1:1);
end

if(~isempty(zipmethod) && numel(item)>minsize)
compfun=str2func([zipmethod 'encode']);
newitem.(N('_ArrayZipType_'))=lower(zipmethod);
Expand All @@ -186,35 +196,51 @@

%%-------------------------------------------------------------------------
function newitem=table2jd(item,varargin)

newitem=struct;
newitem(N('_TableRows_',varargin{:}))=item.Properties.RowNames';
newitem(N('_TableCols_',varargin{:}))=item.Properties.VariableNames;
newitem(N('_TableRecords_',varargin{:}))=table2cell(item);
newitem(N_('_TableRows_',varargin{:}))=item.Properties.RowNames';
newitem(N_('_TableCols_',varargin{:}))=item.Properties.VariableNames;
newitem(N_('_TableRecords_',varargin{:}))=table2cell(item);

%%-------------------------------------------------------------------------
function newitem=graph2jd(item,varargin)

newitem=struct;
nodedata=table2struct(item.Nodes);
if(isfield(nodedata,'Name'))
nodedata=rmfield(nodedata,'Name');
newitem.(N_('_GraphNodes_',varargin{:}))=containers.Map(item.Nodes.Name,num2cell(nodedata),'uniformValues',false);
newitem.(N_('_GraphNodes_',varargin{:}))=containers.Map(item.Nodes.Name,num2cell(nodedata),'UniformValues',false);
else
newitem.(N_('_GraphNodes_',varargin{:}))=containers.Map(1:max(item.Edges.EndNodes(:)),num2cell(nodedata),'uniformValues',false);
newitem.(N_('_GraphNodes_',varargin{:}))=containers.Map(1:max(item.Edges.EndNodes(:)),num2cell(nodedata),'UniformValues',false);
end
edgenodes=item.Edges.EndNodes;
edgenodes=num2cell(item.Edges.EndNodes);
edgedata=table2struct(item.Edges);
if(isfield(edgedata,'EndNodes'))
edgedata=rmfield(edgedata,'EndNodes');
end
edgenodes(:,3)=num2cell(edgedata);
newitem.(N_('_GraphEdges_',varargin{:}))=edgenodes;
if(isa(item,'graph'))
if(strcmp(jsonopt('Prefix',sprintf('x0x%X','_'+0),varargin{:}),'x'))
newitem.(genvarname('_GraphEdges0_'))=edgenodes;
else
newitem.(encodevarname('_GraphEdges0_'))=edgenodes;
end
else
newitem.(N_('_GraphEdges_',varargin{:}))=edgenodes;
end

%%-------------------------------------------------------------------------
function newitem=any2jd(item,varargin)
newitem.(N_('_DataInfo_',varargin{:}))=struct('MATLABObjectClass',class(item),'MATLABObjectSize',size(item));
newitem.(N_('_ByteStream_',varargin{:}))=getByteStreamFromArray(item); % use undocumented matlab function

N=@(x) N_(x,varargin{:});
newitem.(N('_DataInfo_'))=struct('MATLABObjectClass',class(item),'MATLABObjectSize',size(item));
newitem.(N('_ByteStream_'))=getByteStreamFromArray(item); % use undocumented matlab function
if(jsonopt('Base64',0,varargin{:}))
newitem.(N('_ByteStream_'))=char(base64encode(newitem.(N('_ByteStream_'))));
end

%%-------------------------------------------------------------------------
function newname=N_(name,varargin)

prefix=jsonopt('Prefix',sprintf('x0x%X','_'+0),varargin{:});
newname=[prefix name];
6 changes: 4 additions & 2 deletions loadjson.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,9 @@
end
end

%% all functions
%%-------------------------------------------------------------------------
%% helper functions
%%-------------------------------------------------------------------------

function [object, pos,index_esc] = parse_array(inputstr, pos, esc, index_esc, varargin) % JSON array is written in row-major order
pos=parse_char(inputstr, pos, '[');
Expand Down Expand Up @@ -424,7 +426,7 @@
pos=parse_char(inputstr, pos, '}');
if(isstruct(object) && jsonopt('JDataDecode',1,varargin{:})==1)
varargin{:}.Recursive=0;
object=jdatadecode(object,varargin{:});
object=jdatadecode(object,'Base64',1,varargin{:});
end
end

Expand Down
Loading

0 comments on commit 874945f

Please sign in to comment.