Commit 41d183473d4f5d3c2d87c22955b7ce55ce578a41
Merge branch 'master' of http://service.routetopa.eu:7480/WebCompDev/COMPONENTS
Showing
20 changed files
with
242 additions
and
55 deletions
bower_components/JSDataChecker/jsdatachecker.min.js deleted
1 | -function ArrayUtils(){}function DataTypeConverter(){this._fields=[],this._numOfRows=0}function DataTypesUtils(){}function DataTypeHierarchy(){}ArrayUtils.TestAndSet=function(arr,key,object){return"undefined"==typeof arr?null:0==Array.isArray(arr)?null:("undefined"==typeof arr[key]&&(arr[key]=object),arr[key])},ArrayUtils.TestAndInitializeKey=function(obj,key,value){return"undefined"==typeof obj?null:("undefined"==typeof obj[key]&&(obj[key]=value),obj[key])},ArrayUtils.TestAndIncrement=function(arr,key){var exists=arr[key];return"undefined"==typeof exists&&(arr[key]=0),arr[key]++,arr},ArrayUtils.toFieldsArray=function(obj){var fields=[];return ArrayUtils.IteratorOverKeys(obj,function(field,key){field.key=key,fields.push(field)}),fields},ArrayUtils.IteratorOverKeys=function(arr,callback){for(var property in arr)if(arr.hasOwnProperty(property)){var item=arr[property];callback(item,property)}},ArrayUtils.FindMinMax=function(arr,fncompare){var max1=null,max2=null;for(var key in arr)null==max1||fncompare(arr[key],max1.value)?(max2=max1,max1={index:-1,key:key,value:arr[key]}):(null==max2||fncompare(arr[key],max2.value))&&(max2={index:-1,key:key,value:arr[key]});return{first:max1,second:max2}},ArrayUtils.isArray=function(arr){return Array.isArray(arr)?arr.length>0:!1},DataTypeConverter.TYPES={TEXT:{value:0,name:"TEXT"},CODE:{value:1,name:"CODE"},NUMBER:{value:2,name:"NUMBER"},OBJECT:{value:3,name:"OBJECT"},BOOL:{value:5,name:"BOOL"},CONST:{value:6,name:"CONST"},CATEGORY:{value:7,name:"CATEGORY"},DATETIME:{value:8,name:"DATETIME"},EMPTY:{value:101,name:"NULL"}},DataTypeConverter.SUBTYPES={PERCENTAGE:{value:1e3,name:"PERCENTAGE"},LATITUDE:{value:1001,name:"LATITUDE"},LONGITUDE:{value:1002,name:"LONGITUDE"}},DataTypeConverter.prototype=function(){var _analyseDataTypes=function(fields){ArrayUtils.IteratorOverKeys(fields,function(field){if(field._inferredTypes[DataTypeConverter.TYPES.CODE.name]){var confidence=field._inferredTypes[DataTypeConverter.TYPES.CODE.name]/field.numOfItems,_numericalInferredType=field._inferredTypes[DataTypeConverter.TYPES.NUMBER.name];return"undefined"!=typeof _numericalInferredType&&(confidence+=_numericalInferredType/field.numOfItems),field.type=DataTypeConverter.TYPES.CODE.name,void(field.typeConfidence=confidence)}var max=ArrayUtils.FindMinMax(field._inferredTypes,function(curval,lastval){return curval>lastval}),tkey=max.first.key;tkey===DataTypeConverter.TYPES.EMPTY.name&&null!=max.second&&"undefined"!=typeof max.second&&(tkey=max.second.key),field.type=tkey,field.typeConfidence=field._inferredTypes[max.first.key]/field.numOfItems})},_processInferType=function(value){if(null===value||"undefined"==typeof value)return DataTypeConverter.TYPES.EMPTY;if("object"==typeof value)return DataTypeConverter.TYPES.OBJECT;if(/^0[0-9]+$/.test(value))return DataTypeConverter.TYPES.CODE;var isnumber=DataTypesUtils.FilterFloat(value);if(isNaN(isnumber)!==!0)return DataTypeConverter.TYPES.NUMBER;var _date=DataTypesUtils.FilterDateTime(value);return 0==isNaN(_date)&&null!=_date?DataTypeConverter.TYPES.DATETIME:DataTypeConverter.TYPES.TEXT},jsonTraverse=function(json,fieldKeys,callback){var stack=[],numOfRows=0;for(stack.push({item:json,fieldKeyIndex:0});stack.length>0;){var stackTask=stack.pop(),item=stackTask.item,fieldKeyIndex=stackTask.fieldKeyIndex,fieldKey=fieldKeys[fieldKeyIndex];if("*"!=fieldKey||0!=ArrayUtils.isArray(item))if("*"!=fieldKey||1!=ArrayUtils.isArray(item)){var jsonSubtree=item[fieldKey];if(Array.isArray(jsonSubtree))for(var j=0;j<jsonSubtree.length;j++){var jsonItem=jsonSubtree[j];stack.push({item:jsonItem,fieldKeyIndex:fieldKeyIndex+1})}else stack.push({item:jsonSubtree,fieldKeyIndex:fieldKeyIndex+1})}else for(var cell,j=0;j<item.length&&(cell=item[j]);j++)stack.push({item:cell,fieldKeyIndex:fieldKeyIndex}),numOfRows++;else{var sProcessedKeys=fieldKeys.slice(0,fieldKeyIndex).toString();ArrayUtils.IteratorOverKeys(item,function(value,key){var curKey=sProcessedKeys+(sProcessedKeys.length>0?",":"")+key,_value=callback(value,key,curKey,numOfRows);item[key]=_value}),numOfRows++}}};return{constructor:DataTypeConverter,cast:function(metadata,options){return"undefined"!=typeof options&&null!=options||(options={castThresholdConfidence:1,castIfNull:!1}),this.convert(metadata,options)},convert:function(metadata,options){var lastRowIndex=0,numOfRows=0,numOfValues=0,datasetErrors=0,datasetMissingValues=0;return"undefined"!=typeof options&&null!=options||(options={castThresholdConfidence:1,castIfNull:!1}),jsonTraverse(metadata.dataset,metadata.fieldKeys,function(value,key,traversedKeys,rowIndex){var inferredType=metadata.types[traversedKeys];numOfValues++,lastRowIndex!=rowIndex&&(lastRowIndex=rowIndex,numOfRows++),null==value||"undefined"==typeof value||0==(value+"").length;var isCast=inferredType.typeConfidence>=options.castThresholdConfidence;if(inferredType.type==DataTypeConverter.TYPES.NUMBER.name&&isCast){var number=parseFloat(value);return isNaN(number)?(datasetErrors++,value):number}return value}),metadata.qualityIndex.notNullValues=(numOfValues-datasetMissingValues)/numOfValues,metadata.qualityIndex.errors=(numOfValues-datasetErrors)/numOfValues,metadata},inferJsonDataType:function(json,fieldKeys){var stack=[],fieldsType={},numOfRows=0;if("undefined"==typeof fieldKeys)throw"IllegalArgumentException: undefined json path to analyse.";for(stack.push({item:json,fieldKeyIndex:0});stack.length>0;){var stackTask=stack.pop(),item=stackTask.item,fieldKeyIndex=stackTask.fieldKeyIndex,fieldKey=fieldKeys[fieldKeyIndex];if("*"!=fieldKey||0!=ArrayUtils.isArray(item))if("*"==fieldKey&&ArrayUtils.isArray(item))for(var cell,j=0;j<item.length&&(cell=item[j]);j++)stack.push({item:cell,fieldKeyIndex:fieldKeyIndex}),numOfRows++;else{var jsonSubtree=item[fieldKey];if(Array.isArray(jsonSubtree))for(var j=0;j<jsonSubtree.length;j++){var jsonItem=jsonSubtree[j];stack.push({item:jsonItem,fieldKeyIndex:fieldKeyIndex+1})}else stack.push({item:jsonSubtree,fieldKeyIndex:fieldKeyIndex+1})}else{var sProcessedKeys=fieldKeys.slice(0,fieldKeyIndex).toString();ArrayUtils.IteratorOverKeys(item,function(item,key){var inferredType=_processInferType(item),curKey=sProcessedKeys+(0==sProcessedKeys.length?"":",")+key,fieldType=ArrayUtils.TestAndInitializeKey(fieldsType,curKey,{name:curKey,_inferredTypes:[],_inferredValues:[],numOfItems:0});fieldType.numOfItems++,ArrayUtils.TestAndIncrement(fieldType._inferredTypes,inferredType.name),inferredType===DataTypeConverter.TYPES.TEXT&&ArrayUtils.TestAndIncrement(fieldType._inferredValues,item)}),numOfRows++}}_analyseDataTypes(fieldsType);var quality={homogeneity:1,completeness:1,totalNullValues:0,totalValues:0};ArrayUtils.IteratorOverKeys(fieldsType,function(fieldType){quality.totalValues+=fieldType.numOfItems,quality.homogeneity*=fieldType.typeConfidence,fieldType.totalNullValues=0,fieldType._inferredTypes.hasOwnProperty(DataTypeConverter.TYPES.EMPTY.name)&&(fieldType.totalNullValues=fieldType._inferredTypes[DataTypeConverter.TYPES.EMPTY.name],quality.totalNullValues+=fieldType.totalNullValues)}),quality.homogeneity=Math.round(100*quality.homogeneity)/100;var totFullValues=quality.totalValues-quality.totalNullValues;quality.completeness=Math.round(totFullValues/quality.totalValues*100)/100;var warningsTextual="";return ArrayUtils.IteratorOverKeys(fieldsType,function(fieldType){fieldType.errorsDescription="";var description="";if(fieldType.typeConfidence<1){var incorrect=fieldType.numOfItems-fieldType.totalNullValues-fieldType._inferredTypes[fieldType.type];if(incorrect>0){description+="The column <"+fieldType.name+"> has the type <"+fieldType.type+">";var verb=1==incorrect?" value is":" values are";description+=", but "+incorrect+verb+" not a "+fieldType.type}}fieldType.totalNullValues>0&&(description+="The column <"+fieldType.name+"> has "+fieldType.totalNullValues+" EMPTY value",fieldType.totalNullValues>1&&(description+="s")),description.length>0&&(description+="."),fieldType.errorsDescription=description,warningsTextual+=description}),{dataset:json,fieldKeys:fieldKeys,types:fieldsType,qualityIndex:quality,warningsTextual:warningsTextual}},inferDataTypeOfValue:function(value){return _processInferType(value)}}}(),DataTypesUtils.FilterTime=function(value){var expTime=/^[0-9]{2}:[0-9]{2}(:[0-9]{2})?(\+[0-9]{2}:[0-9]{2})?$/;if(0==expTime.test(value))return null;var splitted=value.split(/[:|\+]/),expNumber=/^[0-9]{2}$/,HH=expNumber.test(splitted[0])?parseInt(splitted[0]):0,MM=expNumber.test(splitted[1])?parseInt(splitted[1]):0,SS=splitted.length>=3&&expNumber.test(splitted[2])?parseInt(splitted[2]):0,dt=new Date;return dt.setHours(HH),dt.setMinutes(MM),dt.setSeconds(SS),dt},DataTypesUtils.FilterDateTime=function(value){var _dtSplitted=value.split(/[T|\s]/);if(2==_dtSplitted.length){var dtTime=DataTypesUtils.FilterTime(_dtSplitted[1]);if(null==dtTime)return null;var dtDateTime=DataTypesUtils.FilterDate(_dtSplitted[0],dtTime);return dtDateTime}var dtDate=DataTypesUtils.FilterDate(value);if(null!=dtDate)return dtDate;var dtTime=DataTypesUtils.FilterTime(value);return dtTime},DataTypesUtils.FilterDate=function(value,dtDate){if(null==dtDate&&(dtDate=new Date),/^[0-9][0-9][0-9][0-9]\-[0-9][0-9]$/.test(value)){var year=parseInt(value.substring(0,4)),month=parseInt(value.substring(5));return dtDate.setYear(year),dtDate.setMonth(month),dtDate}var expDate=/^[0-9]{4}(\-|\/)[0-9]{2}((\-|\/)[0-9]{2})?$/;if(expDate.test(value)){var splitted=value.split(/[\-|\/]/),year=parseInt(splitted[0]),month=parseInt(splitted[1]),day=3==splitted.length?parseInt(splitted[2]):0;return dtDate.setYear(year),dtDate.setMonth(month),dtDate.setDate(day),dtDate}if(expDate=/^[0-9]{2}(\-|\/)[0-9]{2}(\-|\/)[0-9]{4}$/,expDate.test(value)){var splitted=value.split(/[\-|\/]/),year=parseInt(splitted[2]),month=parseInt(splitted[1]),day=parseInt(splitted[0]);return dtDate.setYear(year),dtDate.setMonth(month),dtDate.setDate(day),dtDate}return null},DataTypesUtils.FilterFloat=function(value){return/^(\-|\+)?((0|([1-9][0-9]*))(\.[0-9]+)?|Infinity)$/.test(value)?Number(value):NaN},DataTypesUtils.DecimalPlaces=function(num){var match=(""+num).match(/(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/);return match?Math.max(0,(match[1]?match[1].length:0)-(match[2]?+match[2]:0)):0},DataTypeHierarchy.HIERARCHY=[],DataTypeHierarchy.HIERARCHY[DataTypeConverter.TYPES.TEXT.name]=[DataTypeConverter.TYPES.TEXT.name],DataTypeHierarchy.HIERARCHY[DataTypeConverter.TYPES.NUMBER.name]=[DataTypeConverter.TYPES.NUMBER.name,DataTypeConverter.TYPES.TEXT.name],DataTypeHierarchy.HIERARCHY[DataTypeConverter.TYPES.DATETIME.name]=[DataTypeConverter.TYPES.DATETIME.name,DataTypeConverter.TYPES.TEXT.name],DataTypeHierarchy.HIERARCHY[DataTypeConverter.SUBTYPES.LATITUDE.name]=[DataTypeConverter.SUBTYPES.LATITUDE.name,DataTypeConverter.TYPES.NUMBER.name,DataTypeConverter.TYPES.TEXT.name],DataTypeHierarchy.HIERARCHY[DataTypeConverter.SUBTYPES.LONGITUDE.name]=[DataTypeConverter.SUBTYPES.LONGITUDE.name,DataTypeConverter.TYPES.NUMBER.name,DataTypeConverter.TYPES.TEXT.name],DataTypeHierarchy.canConvert=function(fromType,toType){var arrConvertableTypes=DataTypeHierarchy.HIERARCHY[fromType],idx=arrConvertableTypes.indexOf(toType);return idx>=0}; | ||
2 | \ No newline at end of file | 0 | \ No newline at end of file |
bower_components/JSDataChecker/.bower.json renamed to bower_components/jsdatachecker/.bower.json
1 | { | 1 | { |
2 | - "name": "JSDataChecker", | ||
3 | - "version": "0.0.8", | ||
4 | - "homepage": "https://github.com/donpir/JSDataChecker", | 2 | + "name": "jsdatachecker", |
3 | + "version": "0.1.1", | ||
4 | + "homepage": "https://github.com/donpir/jsdatachecker", | ||
5 | "authors": [ | 5 | "authors": [ |
6 | "'Donato Pirozzi <'donatopirozzi@gmail.com'>" | 6 | "'Donato Pirozzi <'donatopirozzi@gmail.com'>" |
7 | ], | 7 | ], |
@@ -22,13 +22,14 @@ | @@ -22,13 +22,14 @@ | ||
22 | "polywc": "https://github.com/donpir/polywc.git#master" | 22 | "polywc": "https://github.com/donpir/polywc.git#master" |
23 | }, | 23 | }, |
24 | "private": false, | 24 | "private": false, |
25 | - "_release": "0.0.8", | 25 | + "_release": "0.1.1", |
26 | "_resolution": { | 26 | "_resolution": { |
27 | "type": "version", | 27 | "type": "version", |
28 | - "tag": "v0.0.8", | ||
29 | - "commit": "862e4d5fe56d629ae7ef8aadb728b1add3af01d9" | 28 | + "tag": "v0.1.1", |
29 | + "commit": "60887fda0d7e7388c5d682228ee1288dd7bcdd09" | ||
30 | }, | 30 | }, |
31 | "_source": "https://github.com/donpir/JSDataChecker.git", | 31 | "_source": "https://github.com/donpir/JSDataChecker.git", |
32 | - "_target": "~0.0.6", | ||
33 | - "_originalSource": "jsdatachecker" | 32 | + "_target": "~0.1.1", |
33 | + "_originalSource": "jsdatachecker", | ||
34 | + "_direct": true | ||
34 | } | 35 | } |
35 | \ No newline at end of file | 36 | \ No newline at end of file |
bower_components/JSDataChecker/LICENSE renamed to bower_components/jsdatachecker/LICENSE
bower_components/JSDataChecker/README.md renamed to bower_components/jsdatachecker/README.md
@@ -3,6 +3,18 @@ JavaScript Library to read dataset content (e.g., Open Data), infer data types, | @@ -3,6 +3,18 @@ JavaScript Library to read dataset content (e.g., Open Data), infer data types, | ||
3 | 3 | ||
4 | In details, it analysis the json (specified paths) to infer the data type. In addition, when request (via code) the library converts the json data based on the inferred type. | 4 | In details, it analysis the json (specified paths) to infer the data type. In addition, when request (via code) the library converts the json data based on the inferred type. |
5 | 5 | ||
6 | +Recognised basic types: | ||
7 | + - Text; | ||
8 | + - Number; | ||
9 | + - Date; | ||
10 | + - Object; | ||
11 | + - Null. | ||
12 | + | ||
13 | +Recognised types: | ||
14 | + - Geographic coordinates; | ||
15 | + - JSON; | ||
16 | + - GEOJSON. | ||
17 | + | ||
6 | ## Documentation | 18 | ## Documentation |
7 | 19 | ||
8 | See wiki pages [here](https://github.com/donpir/JSDataChecker/wiki). | 20 | See wiki pages [here](https://github.com/donpir/JSDataChecker/wiki). |
@@ -13,8 +25,8 @@ The library has released with the LGPL license [link here](http://www.gnu.org/li | @@ -13,8 +25,8 @@ The library has released with the LGPL license [link here](http://www.gnu.org/li | ||
13 | 25 | ||
14 | Briefly with this license: | 26 | Briefly with this license: |
15 | 27 | ||
16 | - - Freedom to modify and distribute modified copies of POLYWC-DATA-TABLE, distributing the source code and releasing it with the same LGPL license; | ||
17 | - - You can dynamically (not statically) link POLYWC-DATA-TABLE library in a proprietary software, keeping the JS files as they are. | 28 | + - Freedom to modify and distribute modified copies of DataChecker, distributing the source code and releasing it with the same LGPL license; |
29 | + - You can dynamically (not statically) link DataChecker library in a proprietary software, keeping the JS files as they are. | ||
18 | 30 | ||
19 | 31 | ||
20 | 32 |
bower_components/JSDataChecker/bower.json renamed to bower_components/jsdatachecker/bower.json
1 | { | 1 | { |
2 | - "name": "JSDataChecker", | ||
3 | - "version": "v0.0.8", | ||
4 | - "homepage": "https://github.com/donpir/JSDataChecker", | 2 | + "name": "jsdatachecker", |
3 | + "version": "v0.1.1", | ||
4 | + "homepage": "https://github.com/donpir/jsdatachecker", | ||
5 | "authors": [ | 5 | "authors": [ |
6 | "'Donato Pirozzi <'donatopirozzi@gmail.com'>" | 6 | "'Donato Pirozzi <'donatopirozzi@gmail.com'>" |
7 | ], | 7 | ], |
bower_components/JSDataChecker/build.cmd renamed to bower_components/jsdatachecker/build.cmd
bower_components/jsdatachecker/jsdatachecker.min.js
0 → 100644
1 | +function ArrayUtils(){}function DataTypeConverter(){this._fields=[],this._numOfRows=0}function DataTypesUtils(){}function DataTypeHierarchy(){}ArrayUtils.TestAndSet=function(arr,key,object){return"undefined"==typeof arr?null:0==Array.isArray(arr)?null:("undefined"==typeof arr[key]&&(arr[key]=object),arr[key])},ArrayUtils.TestAndInitializeKey=function(obj,key,value){return"undefined"==typeof obj?null:("undefined"==typeof obj[key]&&(obj[key]=value),obj[key])},ArrayUtils.TestAndIncrement=function(arr,key){var exists=arr[key];return"undefined"==typeof exists&&(arr[key]=0),arr[key]++,arr},ArrayUtils.toFieldsArray=function(obj){var fields=[];return ArrayUtils.IteratorOverKeys(obj,function(field,key){field.key=key,fields.push(field)}),fields},ArrayUtils.IteratorOverKeys=function(arr,callback){for(var property in arr)if(arr.hasOwnProperty(property)){var item=arr[property];callback(item,property)}},ArrayUtils.FindMinMax=function(arr,fncompare){var max1=null,max2=null;for(var key in arr)null==max1||fncompare(arr[key],max1.value)?(max2=max1,max1={index:-1,key:key,value:arr[key]}):(null==max2||fncompare(arr[key],max2.value))&&(max2={index:-1,key:key,value:arr[key]});return{first:max1,second:max2}},ArrayUtils.isArray=function(arr){return Array.isArray(arr)?arr.length>0:!1},DataTypeConverter.TYPES={TEXT:{value:0,name:"TEXT"},CODE:{value:1,name:"CODE"},NUMBER:{value:2,name:"NUMBER"},OBJECT:{value:3,name:"OBJECT"},BOOL:{value:5,name:"BOOL"},CONST:{value:6,name:"CONST"},CATEGORY:{value:7,name:"CATEGORY"},DATETIME:{value:8,name:"DATETIME"},EMPTY:{value:101,name:"NULL"}},DataTypeConverter.SUBTYPES={GEOCOORDINATE:{value:1e3,name:"GEOCOORDINATE"},GEOJSON:{value:1001,name:"GEOJSON"},PERCENTAGE:{value:1100,name:"PERCENTAGE"},LATITUDE:{value:1101,name:"LATITUDE"},LONGITUDE:{value:1102,name:"LONGITUDE"}},DataTypeConverter.GEOJSONTYPES=["Point","MultiPoint","LineString","MultiLineString","Polygon","MultiPolygon","GeometryCollection"],DataTypeConverter.prototype=function(){var _analyseDataTypes=function(fields){ArrayUtils.IteratorOverKeys(fields,function(field){if(field._inferredTypes[DataTypeConverter.TYPES.CODE.name]){var confidence=field._inferredTypes[DataTypeConverter.TYPES.CODE.name]/field.numOfItems,_numericalInferredType=field._inferredTypes[DataTypeConverter.TYPES.NUMBER.name];return"undefined"!=typeof _numericalInferredType&&(confidence+=_numericalInferredType/field.numOfItems),field.type=DataTypeConverter.TYPES.CODE.name,void(field.typeConfidence=confidence)}var max=ArrayUtils.FindMinMax(field._inferredTypes,function(curval,lastval){return curval>lastval}),tkey=max.first.key;tkey===DataTypeConverter.TYPES.EMPTY.name&&null!=max.second&&"undefined"!=typeof max.second&&(tkey=max.second.key),field.type=tkey,field.typeConfidence=field._inferredTypes[max.first.key]/field.numOfItems;var max=ArrayUtils.FindMinMax(field._inferredSubTypes,function(curval,lastval){return curval>lastval});if(field.subtype=null,null!=max&&null!=max.first){field.subtype=max.first.key,field.subtypeConfidence=field._inferredSubTypes[field.subtype]/field.numOfItems;var fieldName=field.name.toLowerCase(),isLatType=field.subtype===DataTypeConverter.SUBTYPES.LATITUDE.name,fieldNameContainsLat=fieldName.indexOf("lat")>=0,fieldNameContainsLon=fieldName.indexOf("ng")>=0;1==isLatType&&0==fieldNameContainsLat&&1==fieldNameContainsLon&&(field.subtype=DataTypeConverter.SUBTYPES.LONGITUDE.name)}})},_processInferType=function(value){if(null===value||"undefined"==typeof value)return DataTypeConverter.TYPES.EMPTY;if("object"==typeof value)return DataTypeConverter.TYPES.OBJECT;if(/^0[0-9]+$/.test(value))return DataTypeConverter.TYPES.CODE;var isnumber=DataTypesUtils.FilterFloat(value);if(isNaN(isnumber)!==!0)return DataTypeConverter.TYPES.NUMBER;var _date=DataTypesUtils.FilterDateTime(value);return 0==isNaN(_date)&&null!=_date?DataTypeConverter.TYPES.DATETIME:DataTypeConverter.TYPES.TEXT},_processInferSubType=function(value){if(null===value||"undefined"==typeof value)return null;if(Array.isArray(value)&&2==value.length&&NaN!=DataTypesUtils.FilterFloat(value[0])&&NaN!=DataTypesUtils.FilterFloat(value[1])&&DataTypesUtils.DecimalPlaces(value[0])>4&&DataTypesUtils.DecimalPlaces(value[1])>4)return DataTypeConverter.SUBTYPES.GEOCOORDINATE;if("string"==typeof value){var split=value.split(",");if(DataTypesUtils.IsLatLng(split[0])&&DataTypesUtils.IsLatLng(split[1]))return DataTypeConverter.SUBTYPES.GEOCOORDINATE}var isnumber=DataTypesUtils.FilterFloat(value);if(isNaN(isnumber)!==!0)return isnumber>=-90&&90>=isnumber&&DataTypesUtils.DecimalPlaces(isnumber)>=5?DataTypeConverter.SUBTYPES.GEOCOORDINATE:isnumber>=-180&&180>=isnumber&&DataTypesUtils.DecimalPlaces(isnumber)>=5?DataTypeConverter.SUBTYPES.GEOCOORDINATE:null;if("object"==typeof value&&value.hasOwnProperty("type")){var geotype=value.type,isincluded=DataTypeConverter.GEOJSONTYPES.includes(geotype);if(isincluded)return DataTypeConverter.SUBTYPES.GEOJSON}return null},_filterBasedOnThreshold=function(metadata,threshold){return ArrayUtils.IteratorOverKeys(metadata.types,function(fieldType,key){if(!(fieldType.typeConfidence>=threshold)){var arrHierarchyTypes=DataTypeHierarchy.HIERARCHY[fieldType.type];if(null==arrHierarchyTypes)return metadata;var lastFieldType={lastType:arrHierarchyTypes[0],lastTypeCounter:fieldType._inferredTypes[arrHierarchyTypes[0]],typeConfidence:0};lastFieldType.typeConfidence=lastFieldType.lastTypeCounter/fieldType.numOfItems;for(var curType,i=1;i<arrHierarchyTypes.length,curType=arrHierarchyTypes[i];i++){var numItemsOfCurType=fieldType._inferredTypes.hasOwnProperty(curType)?fieldType._inferredTypes[curType]:0;if(lastFieldType.lastType=curType,lastFieldType.lastTypeCounter+=numItemsOfCurType,lastFieldType.typeConfidence=lastFieldType.lastTypeCounter/fieldType.numOfItems,lastFieldType.typeConfidence>=threshold){fieldType.type=lastFieldType.lastType,fieldType.typeConfidence=lastFieldType.typeConfidence;break}}}}),metadata},jsonTraverse=function(json,fieldKeys,callback){var stack=[],numOfRows=0;for(stack.push({item:json,fieldKeyIndex:0});stack.length>0;){var stackTask=stack.pop(),item=stackTask.item,fieldKeyIndex=stackTask.fieldKeyIndex,fieldKey=fieldKeys[fieldKeyIndex];if("*"!=fieldKey||0!=ArrayUtils.isArray(item))if("*"!=fieldKey||1!=ArrayUtils.isArray(item)){var jsonSubtree=item[fieldKey];if(Array.isArray(jsonSubtree))for(var j=0;j<jsonSubtree.length;j++){var jsonItem=jsonSubtree[j];stack.push({item:jsonItem,fieldKeyIndex:fieldKeyIndex+1})}else stack.push({item:jsonSubtree,fieldKeyIndex:fieldKeyIndex+1})}else for(var cell,j=0;j<item.length&&(cell=item[j]);j++)stack.push({item:cell,fieldKeyIndex:fieldKeyIndex}),numOfRows++;else{var sProcessedKeys=fieldKeys.slice(0,fieldKeyIndex).toString();ArrayUtils.IteratorOverKeys(item,function(value,key){var curKey=sProcessedKeys+(sProcessedKeys.length>0?",":"")+key,_value=callback(value,key,curKey,numOfRows);item[key]=_value}),numOfRows++}}};return{constructor:DataTypeConverter,cast:function(metadata,options){return"undefined"!=typeof options&&null!=options||(options={castThresholdConfidence:1,castIfNull:!1}),this.convert(metadata,options)},convert:function(metadata,options){var lastRowIndex=0,numOfRows=0,numOfValues=0,datasetErrors=0,datasetMissingValues=0;return"undefined"!=typeof options&&null!=options||(options={castThresholdConfidence:1,castIfNull:!1}),jsonTraverse(metadata.dataset,metadata.fieldKeys,function(value,key,traversedKeys,rowIndex){var inferredType=metadata.types[traversedKeys];numOfValues++,lastRowIndex!=rowIndex&&(lastRowIndex=rowIndex,numOfRows++),null==value||"undefined"==typeof value||0==(value+"").length;var isCast=inferredType.typeConfidence>=options.castThresholdConfidence;if(inferredType.type==DataTypeConverter.TYPES.NUMBER.name&&isCast){var number=parseFloat(value);return isNaN(number)?(datasetErrors++,value):number}return value}),metadata.qualityIndex.notNullValues=(numOfValues-datasetMissingValues)/numOfValues,metadata.qualityIndex.errors=(numOfValues-datasetErrors)/numOfValues,metadata},inferJsonDataType:function(json,fieldKeys,options){"undefined"!=typeof options&&null!=options||(options={thresholdConfidence:1});var stack=[],fieldsType={},numOfRows=0;if("undefined"==typeof fieldKeys)throw"IllegalArgumentException: undefined json path to analyse.";for(stack.push({item:json,fieldKeyIndex:0});stack.length>0;){var stackTask=stack.pop(),item=stackTask.item,fieldKeyIndex=stackTask.fieldKeyIndex,fieldKey=fieldKeys[fieldKeyIndex];if("*"!=fieldKey||0!=ArrayUtils.isArray(item))if("*"==fieldKey&&ArrayUtils.isArray(item))for(var cell,j=0;j<item.length&&(cell=item[j]);j++)stack.push({item:cell,fieldKeyIndex:fieldKeyIndex}),numOfRows++;else{var jsonSubtree=item[fieldKey];if(Array.isArray(jsonSubtree))for(var j=0;j<jsonSubtree.length;j++){var jsonItem=jsonSubtree[j];stack.push({item:jsonItem,fieldKeyIndex:fieldKeyIndex+1})}else stack.push({item:jsonSubtree,fieldKeyIndex:fieldKeyIndex+1})}else{var sProcessedKeys=fieldKeys.slice(0,fieldKeyIndex).toString();ArrayUtils.IteratorOverKeys(item,function(item,key){var curKey=sProcessedKeys+(0==sProcessedKeys.length?"":",")+key,fieldType=ArrayUtils.TestAndInitializeKey(fieldsType,curKey,{name:curKey,_inferredTypes:[],_inferredSubTypes:[],_inferredValues:[],numOfItems:0});fieldType.numOfItems++;var inferredType=_processInferType(item);ArrayUtils.TestAndIncrement(fieldType._inferredTypes,inferredType.name),inferredType===DataTypeConverter.TYPES.TEXT&&ArrayUtils.TestAndIncrement(fieldType._inferredValues,item);var inferredSubType=_processInferSubType(item);null!=inferredSubType&&"undefined"!=typeof inferredSubType&&ArrayUtils.TestAndIncrement(fieldType._inferredSubTypes,inferredSubType.name)}),numOfRows++}}_analyseDataTypes(fieldsType);var quality={homogeneity:1,completeness:1,totalNullValues:0,totalValues:0};ArrayUtils.IteratorOverKeys(fieldsType,function(fieldType){quality.totalValues+=fieldType.numOfItems,quality.homogeneity*=fieldType.typeConfidence,fieldType.totalNullValues=0,fieldType._inferredTypes.hasOwnProperty(DataTypeConverter.TYPES.EMPTY.name)&&(fieldType.totalNullValues=fieldType._inferredTypes[DataTypeConverter.TYPES.EMPTY.name],quality.totalNullValues+=fieldType.totalNullValues)}),quality.homogeneity=Math.round(100*quality.homogeneity)/100;var totFullValues=quality.totalValues-quality.totalNullValues;quality.completeness=Math.round(totFullValues/quality.totalValues*100)/100;var warningsTextual="";ArrayUtils.IteratorOverKeys(fieldsType,function(fieldType){fieldType.errorsDescription="";var description="";if(fieldType.typeConfidence<1){var incorrect=fieldType.numOfItems-fieldType.totalNullValues-fieldType._inferredTypes[fieldType.type];if(incorrect>0){description+="The column <"+fieldType.name+"> has the type <"+fieldType.type+">";var verb=1==incorrect?" value is":" values are";description+=", but "+incorrect+verb+" not a "+fieldType.type}}fieldType.totalNullValues>0&&(description+="The column <"+fieldType.name+"> has "+fieldType.totalNullValues+" EMPTY value",fieldType.totalNullValues>1&&(description+="s")),description.length>0&&(description+="."),fieldType.errorsDescription=description,warningsTextual+=description});var metadata={dataset:json,fieldKeys:fieldKeys,types:fieldsType,qualityIndex:quality,warningsTextual:warningsTextual};return _filterBasedOnThreshold(metadata,options.thresholdConfidence),metadata},inferDataTypeOfValue:function(value){return _processInferType(value)},inferDataSubTypeOfValue:function(value){return _processInferSubType(value)}}}(),DataTypesUtils.FilterTime=function(value){var expTime=/^[0-9]{2}:[0-9]{2}(:[0-9]{2})?(\+[0-9]{2}:[0-9]{2})?$/;if(0==expTime.test(value))return null;var splitted=value.split(/[:|\+]/),expNumber=/^[0-9]{2}$/,HH=expNumber.test(splitted[0])?parseInt(splitted[0]):0,MM=expNumber.test(splitted[1])?parseInt(splitted[1]):0,SS=splitted.length>=3&&expNumber.test(splitted[2])?parseInt(splitted[2]):0,dt=new Date;return dt.setHours(HH),dt.setMinutes(MM),dt.setSeconds(SS),dt},DataTypesUtils.FilterDateTime=function(value){var _dtSplitted=value.split(/[T|\s]/);if(2==_dtSplitted.length){var dtTime=DataTypesUtils.FilterTime(_dtSplitted[1]);if(null==dtTime)return null;var dtDateTime=DataTypesUtils.FilterDate(_dtSplitted[0],dtTime);return dtDateTime}var dtDate=DataTypesUtils.FilterDate(value);if(null!=dtDate)return dtDate;var dtTime=DataTypesUtils.FilterTime(value);return dtTime},DataTypesUtils.FilterDate=function(value,dtDate){if(null==dtDate&&(dtDate=new Date),/^[0-9][0-9][0-9][0-9]\-[0-9][0-9]$/.test(value)){var year=parseInt(value.substring(0,4)),month=parseInt(value.substring(5));return dtDate.setYear(year),dtDate.setMonth(month),dtDate}var expDate=/^[0-9]{4}(\-|\/)[0-9]{2}((\-|\/)[0-9]{2})?$/;if(expDate.test(value)){var splitted=value.split(/[\-|\/]/),year=parseInt(splitted[0]),month=parseInt(splitted[1]),day=3==splitted.length?parseInt(splitted[2]):0;return dtDate.setYear(year),dtDate.setMonth(month),dtDate.setDate(day),dtDate}if(expDate=/^[0-9]{2}(\-|\/)[0-9]{2}(\-|\/)[0-9]{4}$/,expDate.test(value)){var splitted=value.split(/[\-|\/]/),year=parseInt(splitted[2]),month=parseInt(splitted[1]),day=parseInt(splitted[0]);return dtDate.setYear(year),dtDate.setMonth(month),dtDate.setDate(day),dtDate}return null},DataTypesUtils.FilterFloat=function(value){return/^(\-|\+)?((0|([1-9][0-9]*))(\.[0-9]+)?|Infinity)$/.test(value)?Number(value):NaN},DataTypesUtils.DecimalPlaces=function(num){var match=(""+num).match(/(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/);return match?Math.max(0,(match[1]?match[1].length:0)-(match[2]?+match[2]:0)):0},DataTypesUtils.IsLatLng=function(num){return NaN==DataTypesUtils.FilterFloat(num)?!1:DataTypesUtils.DecimalPlaces(num)>4},DataTypeHierarchy.HIERARCHY=[],DataTypeHierarchy.HIERARCHY[DataTypeConverter.TYPES.TEXT.name]=[DataTypeConverter.TYPES.TEXT.name],DataTypeHierarchy.HIERARCHY[DataTypeConverter.TYPES.NUMBER.name]=[DataTypeConverter.TYPES.NUMBER.name,DataTypeConverter.TYPES.TEXT.name],DataTypeHierarchy.HIERARCHY[DataTypeConverter.TYPES.DATETIME.name]=[DataTypeConverter.TYPES.DATETIME.name,DataTypeConverter.TYPES.TEXT.name],DataTypeHierarchy.HIERARCHY[DataTypeConverter.SUBTYPES.GEOCOORDINATE.name]=[DataTypeConverter.SUBTYPES.GEOCOORDINATE.name,DataTypeConverter.TYPES.NUMBER.name,DataTypeConverter.TYPES.TEXT.name],DataTypeHierarchy.canConvert=function(fromType,toType){var arrConvertableTypes=DataTypeHierarchy.HIERARCHY[fromType],idx=arrConvertableTypes.indexOf(toType);return idx>=0}; | ||
0 | \ No newline at end of file | 2 | \ No newline at end of file |
bower_components/JSDataChecker/src/ArrayUtils.js renamed to bower_components/jsdatachecker/src/ArrayUtils.js
bower_components/JSDataChecker/src/DataTypeConverter.js renamed to bower_components/jsdatachecker/src/DataTypeConverter.js
@@ -42,11 +42,17 @@ DataTypeConverter.TYPES = { | @@ -42,11 +42,17 @@ DataTypeConverter.TYPES = { | ||
42 | }; | 42 | }; |
43 | 43 | ||
44 | DataTypeConverter.SUBTYPES = { | 44 | DataTypeConverter.SUBTYPES = { |
45 | - PERCENTAGE : { value: 1000, name: "PERCENTAGE" }, | ||
46 | - LATITUDE : { value: 1001, name: "LATITUDE" }, | ||
47 | - LONGITUDE : { value: 1002, name: "LONGITUDE" } | 45 | + GEOCOORDINATE : { value: 1000, name: "GEOCOORDINATE" }, |
46 | + GEOJSON : { value: 1001, name: "GEOJSON" }, | ||
47 | + | ||
48 | + PERCENTAGE : { value: 1100, name: "PERCENTAGE" }, | ||
49 | + LATITUDE : { value: 1101, name: "LATITUDE" }, | ||
50 | + LONGITUDE : { value: 1102, name: "LONGITUDE" } | ||
48 | }; | 51 | }; |
49 | 52 | ||
53 | +DataTypeConverter.GEOJSONTYPES = [ "Point", "MultiPoint", "LineString", | ||
54 | + "MultiLineString", "Polygon", "MultiPolygon", "GeometryCollection" ]; | ||
55 | + | ||
50 | DataTypeConverter.prototype = (function () { | 56 | DataTypeConverter.prototype = (function () { |
51 | 57 | ||
52 | /*** | 58 | /*** |
@@ -120,7 +126,6 @@ DataTypeConverter.prototype = (function () { | @@ -120,7 +126,6 @@ DataTypeConverter.prototype = (function () { | ||
120 | return; | 126 | return; |
121 | } | 127 | } |
122 | 128 | ||
123 | - | ||
124 | //Infers the field TYPE. | 129 | //Infers the field TYPE. |
125 | var max = ArrayUtils.FindMinMax(field._inferredTypes, function (curval, lastval) { | 130 | var max = ArrayUtils.FindMinMax(field._inferredTypes, function (curval, lastval) { |
126 | return curval > lastval; | 131 | return curval > lastval; |
@@ -136,15 +141,31 @@ DataTypeConverter.prototype = (function () { | @@ -136,15 +141,31 @@ DataTypeConverter.prototype = (function () { | ||
136 | field.typeConfidence = field._inferredTypes[max.first.key] / field.numOfItems; | 141 | field.typeConfidence = field._inferredTypes[max.first.key] / field.numOfItems; |
137 | 142 | ||
138 | 143 | ||
139 | - //TODO: improve this piece of code. | ||
140 | - //LAT/LNG. | ||
141 | - /*var fieldName = field.name.toLowerCase(); | ||
142 | - var isLatType = (field.type === DataTypeConverter.TYPES.LATITUDE.name); | ||
143 | - var fieldNameContainsLat = fieldName.indexOf('lat') >= 0; | ||
144 | - var fieldNameContainsLon = fieldName.indexOf('ng') >= 0; //It could be 'lng'. | ||
145 | - if (isLatType == true && fieldNameContainsLat == false && fieldNameContainsLon == true) { | ||
146 | - field.type = DataTypeConverter.TYPES.LONGITUDE.name; | ||
147 | - }*/ | 144 | + //########## |
145 | + //Infers the field SUBTYPE. | ||
146 | + | ||
147 | + var max = ArrayUtils.FindMinMax(field._inferredSubTypes, function (curval, lastval) { | ||
148 | + return curval > lastval; | ||
149 | + }); | ||
150 | + field.subtype = null; | ||
151 | + if (max != null && max.first != null) { | ||
152 | + field.subtype = max.first.key; | ||
153 | + field.subtypeConfidence = field._inferredSubTypes[field.subtype] / field.numOfItems; | ||
154 | + | ||
155 | + //TODO: improve this piece of code. | ||
156 | + //LAT/LNG. | ||
157 | + var fieldName = field.name.toLowerCase(); | ||
158 | + var isLatType = (field.subtype === DataTypeConverter.SUBTYPES.LATITUDE.name); | ||
159 | + var fieldNameContainsLat = fieldName.indexOf('lat') >= 0; | ||
160 | + var fieldNameContainsLon = fieldName.indexOf('ng') >= 0; //It could be 'lng'. | ||
161 | + if (isLatType == true && fieldNameContainsLat == false && fieldNameContainsLon == true) { | ||
162 | + field.subtype = DataTypeConverter.SUBTYPES.LONGITUDE.name; | ||
163 | + } | ||
164 | + } | ||
165 | + | ||
166 | + /// | ||
167 | + /// SUBTYPES. | ||
168 | + | ||
148 | 169 | ||
149 | //BOOLEAN. | 170 | //BOOLEAN. |
150 | /*var numOfValues = Object.keys(field._inferredValues).length; | 171 | /*var numOfValues = Object.keys(field._inferredValues).length; |
@@ -200,13 +221,83 @@ DataTypeConverter.prototype = (function () { | @@ -200,13 +221,83 @@ DataTypeConverter.prototype = (function () { | ||
200 | if (isNaN(_date) == false && _date != null) | 221 | if (isNaN(_date) == false && _date != null) |
201 | return DataTypeConverter.TYPES.DATETIME; | 222 | return DataTypeConverter.TYPES.DATETIME; |
202 | 223 | ||
203 | - | ||
204 | return DataTypeConverter.TYPES.TEXT; | 224 | return DataTypeConverter.TYPES.TEXT; |
205 | };//EndFunction. | 225 | };//EndFunction. |
206 | 226 | ||
207 | - var _processInferSubType = function(value) { | ||
208 | - //Try to parse lat/lng. | 227 | + var _processInferSubType = function (value) { |
228 | + if (value === null || typeof value === 'undefined') return null; | ||
229 | + | ||
230 | + //GEOCOORDINATE | ||
231 | + if (Array.isArray(value) && value.length == 2) {//It recognises the LAT LNG as array of two values. | ||
232 | + //Checks if the two array's values are numbers. | ||
233 | + if ( DataTypesUtils.FilterFloat(value[0]) != NaN && DataTypesUtils.FilterFloat(value[1]) != NaN ) | ||
234 | + if (DataTypesUtils.DecimalPlaces(value[0]) > 4 && DataTypesUtils.DecimalPlaces(value[1]) > 4 ) | ||
235 | + return DataTypeConverter.SUBTYPES.GEOCOORDINATE; | ||
236 | + }//EndIf. | ||
237 | + | ||
238 | + if (typeof value === 'string') { | ||
239 | + var split = value.split(","); | ||
240 | + if (DataTypesUtils.IsLatLng(split[0]) && DataTypesUtils.IsLatLng(split[1])) | ||
241 | + return DataTypeConverter.SUBTYPES.GEOCOORDINATE; | ||
242 | + } | ||
209 | 243 | ||
244 | + //Try to parse the float. | ||
245 | + var isnumber = DataTypesUtils.FilterFloat(value); | ||
246 | + if (isNaN(isnumber) !== true) {//It is a number. | ||
247 | + //If the number ranges from -90.0 to 90.0, the value is marked as Latitude. | ||
248 | + if (-90.0 <= isnumber && isnumber <= 90.0 && DataTypesUtils.DecimalPlaces(isnumber) >= 5) | ||
249 | + return DataTypeConverter.SUBTYPES.GEOCOORDINATE; | ||
250 | + | ||
251 | + //It the number ranges from -180.0 to 180.0, the value is marked as Longitude. | ||
252 | + if (-180.0 <= isnumber && isnumber <= 180.0 && DataTypesUtils.DecimalPlaces(isnumber) >= 5) | ||
253 | + return DataTypeConverter.SUBTYPES.GEOCOORDINATE; | ||
254 | + | ||
255 | + /*if (0.0 <= isnumber && isnumber <= 100.0) | ||
256 | + if(/^(\+)?((0|([1-9][0-9]*))\.([0-9]+))$/ .test(value)) | ||
257 | + return DataTypeConverter.SUBTYPES.PERCENTAGE;*/ | ||
258 | + | ||
259 | + return null; | ||
260 | + } | ||
261 | + | ||
262 | + //Try to parse GEOJSON. | ||
263 | + if (typeof value === 'object' && value.hasOwnProperty('type')) { | ||
264 | + //Check the type variable. | ||
265 | + var geotype = value.type; | ||
266 | + var isincluded = DataTypeConverter.GEOJSONTYPES.includes(geotype); | ||
267 | + if (isincluded) return DataTypeConverter.SUBTYPES.GEOJSON; | ||
268 | + } | ||
269 | + | ||
270 | + return null; | ||
271 | + };//EndFunction. | ||
272 | + | ||
273 | + var _filterBasedOnThreshold = function(metadata, threshold) { | ||
274 | + ArrayUtils.IteratorOverKeys(metadata.types, function (fieldType, key) { | ||
275 | + if (fieldType.typeConfidence >= threshold) return; | ||
276 | + | ||
277 | + var arrHierarchyTypes = DataTypeHierarchy.HIERARCHY[fieldType.type]; | ||
278 | + if (arrHierarchyTypes == null) | ||
279 | + return metadata; | ||
280 | + | ||
281 | + var lastFieldType = { lastType: arrHierarchyTypes[0], | ||
282 | + lastTypeCounter: fieldType._inferredTypes[arrHierarchyTypes[0]], | ||
283 | + typeConfidence: 0 }; | ||
284 | + lastFieldType.typeConfidence = lastFieldType.lastTypeCounter / fieldType.numOfItems; | ||
285 | + | ||
286 | + for (var i= 1, curType; i<arrHierarchyTypes.length, curType = arrHierarchyTypes[i]; i++) { | ||
287 | + var numItemsOfCurType = fieldType._inferredTypes.hasOwnProperty(curType) ? fieldType._inferredTypes[curType] : 0 ; | ||
288 | + lastFieldType.lastType = curType; | ||
289 | + lastFieldType.lastTypeCounter += numItemsOfCurType; | ||
290 | + lastFieldType.typeConfidence = lastFieldType.lastTypeCounter / fieldType.numOfItems; | ||
291 | + | ||
292 | + if (lastFieldType.typeConfidence >= threshold) { | ||
293 | + fieldType.type = lastFieldType.lastType; | ||
294 | + fieldType.typeConfidence = lastFieldType.typeConfidence; | ||
295 | + break; | ||
296 | + } | ||
297 | + } | ||
298 | + }); | ||
299 | + | ||
300 | + return metadata; | ||
210 | };//EndFunction. | 301 | };//EndFunction. |
211 | 302 | ||
212 | var jsonTraverse = function(json, fieldKeys, callback) { | 303 | var jsonTraverse = function(json, fieldKeys, callback) { |
@@ -332,10 +423,15 @@ DataTypeConverter.prototype = (function () { | @@ -332,10 +423,15 @@ DataTypeConverter.prototype = (function () { | ||
332 | * It parses the json and infers the data types. | 423 | * It parses the json and infers the data types. |
333 | * @param json | 424 | * @param json |
334 | * @param path Array of field keys/names. | 425 | * @param path Array of field keys/names. |
426 | + * @param options Infer Data Type options, in particular the threshold value for the confidence. | ||
335 | */ | 427 | */ |
336 | - inferJsonDataType: function (json, fieldKeys) { | 428 | + inferJsonDataType: function (json, fieldKeys, options) { |
429 | + if (typeof options === 'undefined' || options == null) | ||
430 | + options = { thresholdConfidence: 1 }; | ||
431 | + | ||
337 | var stack = []; | 432 | var stack = []; |
338 | var fieldsType = {}; | 433 | var fieldsType = {}; |
434 | + var fieldsSubType = {}; | ||
339 | var numOfRows = 0; | 435 | var numOfRows = 0; |
340 | 436 | ||
341 | if (typeof fieldKeys == 'undefined') | 437 | if (typeof fieldKeys == 'undefined') |
@@ -357,16 +453,25 @@ DataTypeConverter.prototype = (function () { | @@ -357,16 +453,25 @@ DataTypeConverter.prototype = (function () { | ||
357 | var sProcessedKeys = fieldKeys.slice(0, fieldKeyIndex).toString(); | 453 | var sProcessedKeys = fieldKeys.slice(0, fieldKeyIndex).toString(); |
358 | 454 | ||
359 | ArrayUtils.IteratorOverKeys(item, function (item, key) { | 455 | ArrayUtils.IteratorOverKeys(item, function (item, key) { |
360 | - var inferredType = _processInferType(item); | ||
361 | var curKey = sProcessedKeys + ((sProcessedKeys.length == 0) ? "" : ",") + key; | 456 | var curKey = sProcessedKeys + ((sProcessedKeys.length == 0) ? "" : ",") + key; |
362 | - | ||
363 | - var fieldType = ArrayUtils.TestAndInitializeKey(fieldsType, curKey, { name: curKey, _inferredTypes: [], _inferredValues: [], numOfItems: 0 }); | 457 | + var fieldType = ArrayUtils.TestAndInitializeKey(fieldsType, curKey, { name: curKey, _inferredTypes: [], _inferredSubTypes: [], _inferredValues: [], numOfItems: 0 }); |
364 | fieldType.numOfItems++; | 458 | fieldType.numOfItems++; |
459 | + | ||
460 | + ///TYPE | ||
461 | + var inferredType = _processInferType(item); | ||
365 | ArrayUtils.TestAndIncrement(fieldType._inferredTypes, inferredType.name); | 462 | ArrayUtils.TestAndIncrement(fieldType._inferredTypes, inferredType.name); |
366 | if (inferredType === DataTypeConverter.TYPES.TEXT) | 463 | if (inferredType === DataTypeConverter.TYPES.TEXT) |
367 | ArrayUtils.TestAndIncrement(fieldType._inferredValues, item); | 464 | ArrayUtils.TestAndIncrement(fieldType._inferredValues, item); |
368 | - //if (inferredType === DataTypeConverter.TYPES.LATITUDE || inferredType === DataTypeConverter.TYPES.LONGITUDE) | ||
369 | - // ArrayUtils.TestAndIncrement(fieldType._inferredTypes, DataTypeConverter.TYPES.NUMBER); | 465 | + |
466 | + ///SUBTYPE | ||
467 | + var inferredSubType = _processInferSubType(item); | ||
468 | + if (inferredSubType != null && typeof inferredSubType !== 'undefined') { | ||
469 | + ArrayUtils.TestAndIncrement(fieldType._inferredSubTypes, inferredSubType.name); | ||
470 | + /*if (inferredSubType === DataTypeConverter.TYPES.LATITUDE) | ||
471 | + ArrayUtils.TestAndIncrement(fieldType._inferredSubTypes, DataTypeConverter.TYPES.LATITUDE); | ||
472 | + if (inferredSubType === DataTypeConverter.TYPES.LONGITUDE) | ||
473 | + ArrayUtils.TestAndIncrement(fieldType._inferredSubTypes, DataTypeConverter.TYPES.LONGITUDE);*/ | ||
474 | + }//EndSubtype. | ||
370 | 475 | ||
371 | }); | 476 | }); |
372 | 477 | ||
@@ -451,7 +556,11 @@ DataTypeConverter.prototype = (function () { | @@ -451,7 +556,11 @@ DataTypeConverter.prototype = (function () { | ||
451 | warningsTextual += description; | 556 | warningsTextual += description; |
452 | }); | 557 | }); |
453 | 558 | ||
454 | - return { dataset: json, fieldKeys: fieldKeys, types: fieldsType, qualityIndex: quality, warningsTextual: warningsTextual }; | 559 | + var metadata = { dataset: json, fieldKeys: fieldKeys, types: fieldsType, qualityIndex: quality, warningsTextual: warningsTextual }; |
560 | + | ||
561 | + _filterBasedOnThreshold(metadata, options.thresholdConfidence); | ||
562 | + | ||
563 | + return metadata; | ||
455 | },//EndFunction. | 564 | },//EndFunction. |
456 | 565 | ||
457 | /*inferDataTypes: function (jsonRows) { | 566 | /*inferDataTypes: function (jsonRows) { |
@@ -468,6 +577,15 @@ DataTypeConverter.prototype = (function () { | @@ -468,6 +577,15 @@ DataTypeConverter.prototype = (function () { | ||
468 | */ | 577 | */ |
469 | inferDataTypeOfValue: function (value) { | 578 | inferDataTypeOfValue: function (value) { |
470 | return _processInferType(value); | 579 | return _processInferType(value); |
580 | + },//EndFunction. | ||
581 | + | ||
582 | + /** | ||
583 | + * Given in input a value, the function infers the data type. | ||
584 | + * @param value | ||
585 | + * @returns {*} | ||
586 | + */ | ||
587 | + inferDataSubTypeOfValue: function (value) { | ||
588 | + return _processInferSubType(value); | ||
471 | }//EndFunction. | 589 | }//EndFunction. |
472 | 590 | ||
473 | }; | 591 | }; |
bower_components/JSDataChecker/src/DataTypeHierarchy.js renamed to bower_components/jsdatachecker/src/DataTypeHierarchy.js
@@ -29,9 +29,7 @@ DataTypeHierarchy.HIERARCHY[DataTypeConverter.TYPES.TEXT.name] = [ DataTypeConve | @@ -29,9 +29,7 @@ DataTypeHierarchy.HIERARCHY[DataTypeConverter.TYPES.TEXT.name] = [ DataTypeConve | ||
29 | DataTypeHierarchy.HIERARCHY[DataTypeConverter.TYPES.NUMBER.name] = [ DataTypeConverter.TYPES.NUMBER.name, DataTypeConverter.TYPES.TEXT.name]; | 29 | DataTypeHierarchy.HIERARCHY[DataTypeConverter.TYPES.NUMBER.name] = [ DataTypeConverter.TYPES.NUMBER.name, DataTypeConverter.TYPES.TEXT.name]; |
30 | DataTypeHierarchy.HIERARCHY[DataTypeConverter.TYPES.DATETIME.name] = [ DataTypeConverter.TYPES.DATETIME.name, DataTypeConverter.TYPES.TEXT.name ]; | 30 | DataTypeHierarchy.HIERARCHY[DataTypeConverter.TYPES.DATETIME.name] = [ DataTypeConverter.TYPES.DATETIME.name, DataTypeConverter.TYPES.TEXT.name ]; |
31 | 31 | ||
32 | -DataTypeHierarchy.HIERARCHY[DataTypeConverter.SUBTYPES.LATITUDE.name] = [ DataTypeConverter.SUBTYPES.LATITUDE.name, | ||
33 | - DataTypeConverter.TYPES.NUMBER.name, DataTypeConverter.TYPES.TEXT.name]; | ||
34 | -DataTypeHierarchy.HIERARCHY[DataTypeConverter.SUBTYPES.LONGITUDE.name] = [ DataTypeConverter.SUBTYPES.LONGITUDE.name, | 32 | +DataTypeHierarchy.HIERARCHY[DataTypeConverter.SUBTYPES.GEOCOORDINATE.name] = [ DataTypeConverter.SUBTYPES.GEOCOORDINATE.name, |
35 | DataTypeConverter.TYPES.NUMBER.name, DataTypeConverter.TYPES.TEXT.name]; | 33 | DataTypeConverter.TYPES.NUMBER.name, DataTypeConverter.TYPES.TEXT.name]; |
36 | 34 | ||
37 | DataTypeHierarchy.canConvert = function (fromType, toType) { | 35 | DataTypeHierarchy.canConvert = function (fromType, toType) { |
bower_components/JSDataChecker/src/DataTypesUtils.js renamed to bower_components/jsdatachecker/src/DataTypesUtils.js
@@ -122,4 +122,10 @@ DataTypesUtils.DecimalPlaces = function (num) { | @@ -122,4 +122,10 @@ DataTypesUtils.DecimalPlaces = function (num) { | ||
122 | (match[1] ? match[1].length : 0) | 122 | (match[1] ? match[1].length : 0) |
123 | // Adjust for scientific notation. | 123 | // Adjust for scientific notation. |
124 | - (match[2] ? +match[2] : 0)); | 124 | - (match[2] ? +match[2] : 0)); |
125 | +}//EndFunction. | ||
126 | + | ||
127 | +DataTypesUtils.IsLatLng = function (num) { | ||
128 | + if (DataTypesUtils.FilterFloat(num) == NaN) return false; | ||
129 | + if (DataTypesUtils.DecimalPlaces(num) > 4) return true; | ||
130 | + return false; | ||
125 | }//EndFunction. | 131 | }//EndFunction. |
126 | \ No newline at end of file | 132 | \ No newline at end of file |
bower_components/JSDataChecker/src/ODPlatforms/CKAN.js renamed to bower_components/jsdatachecker/src/ODPlatforms/CKAN.js
bower_components/JSDataChecker/src/ODPlatforms/ODStatistics.js renamed to bower_components/jsdatachecker/src/ODPlatforms/ODStatistics.js
controllets/cocreation-paper-card-controllet/cocreation-paper-card-controllet.html
1 | <link rel="import" href="../../bower_components/polymer/polymer.html"/> | 1 | <link rel="import" href="../../bower_components/polymer/polymer.html"/> |
2 | <link rel="import" href="../../bower_components/paper-card/paper-card.html"/> | 2 | <link rel="import" href="../../bower_components/paper-card/paper-card.html"/> |
3 | +<link rel="import" href="../../bower_components/paper-fab/paper-fab.html"> | ||
3 | <link rel="import" href="../../bower_components/paper-icon-button/paper-icon-button.html"/> | 4 | <link rel="import" href="../../bower_components/paper-icon-button/paper-icon-button.html"/> |
4 | <link rel="import" href="../../bower_components/paper-button/paper-button.html"/> | 5 | <link rel="import" href="../../bower_components/paper-button/paper-button.html"/> |
5 | <link rel="import" href="../../bower_components/iron-icon/iron-icon.html"/> | 6 | <link rel="import" href="../../bower_components/iron-icon/iron-icon.html"/> |
@@ -67,6 +68,13 @@ | @@ -67,6 +68,13 @@ | ||
67 | min-width: 310px; | 68 | min-width: 310px; |
68 | min-height: 188px; | 69 | min-height: 188px; |
69 | } | 70 | } |
71 | + .badge{ | ||
72 | + position: absolute; | ||
73 | + right: 5px; | ||
74 | + bottom: 5px; | ||
75 | + background: #2196F3; | ||
76 | + } | ||
77 | + | ||
70 | </style> | 78 | </style> |
71 | 79 | ||
72 | 80 | ||
@@ -80,12 +88,17 @@ | @@ -80,12 +88,17 @@ | ||
80 | <span class="from">{{from}}</span> | 88 | <span class="from">{{from}}</span> |
81 | <iron-icon class="icon-to" icon="alarm-off"></iron-icon> | 89 | <iron-icon class="icon-to" icon="alarm-off"></iron-icon> |
82 | <span class="to">{{to}}</span> | 90 | <span class="to">{{to}}</span> |
91 | + <template is="dom-if" if="{{!checkRoomType(roomType)}}"> | ||
92 | + <paper-fab mini disabled icon="assessment" class="badge data"></paper-fab> | ||
93 | + </template> | ||
94 | + <template is="dom-if" if="{{checkRoomType(roomType)}}"> | ||
95 | + <paper-fab mini disabled icon="description" class="badge knowledge"></paper-fab> | ||
96 | + </template> | ||
83 | </div> | 97 | </div> |
84 | </div> | 98 | </div> |
85 | <div class="card-actions"> | 99 | <div class="card-actions"> |
86 | <paper-button on-click="_onExplore">Explore</paper-button> | 100 | <paper-button on-click="_onExplore">Explore</paper-button> |
87 | - <paper-icon-button icon="expand-less" on-click="_toggle" style="float:right;"> | ||
88 | - </paper-icon-button> | 101 | + <paper-icon-button icon="expand-less" on-click="_toggle" style="float:right;"></paper-icon-button> |
89 | <iron-collapse class="iron-collapse-over" id="more_info" no-animation> | 102 | <iron-collapse class="iron-collapse-over" id="more_info" no-animation> |
90 | <div class="collapse-content"> | 103 | <div class="collapse-content"> |
91 | {{goal}} | 104 | {{goal}} |
@@ -127,6 +140,10 @@ | @@ -127,6 +140,10 @@ | ||
127 | roomUrl: { | 140 | roomUrl: { |
128 | type: String, | 141 | type: String, |
129 | value: undefined | 142 | value: undefined |
143 | + }, | ||
144 | + roomType:{ | ||
145 | + type: String, | ||
146 | + value: undefined | ||
130 | } | 147 | } |
131 | }, | 148 | }, |
132 | 149 | ||
@@ -142,8 +159,14 @@ | @@ -142,8 +159,14 @@ | ||
142 | 159 | ||
143 | _onExplore: function(e){ | 160 | _onExplore: function(e){ |
144 | window.location = this.roomUrl; | 161 | window.location = this.roomUrl; |
145 | - } | 162 | + }, |
146 | 163 | ||
164 | + checkRoomType: function(type){ | ||
165 | + if(type == "knowledge") | ||
166 | + return true; | ||
167 | + else | ||
168 | + return false; | ||
169 | + } | ||
147 | }) | 170 | }) |
148 | </script> | 171 | </script> |
149 | 172 |
controllets/data-table-controllet/data-table-controllet.html
@@ -4,9 +4,10 @@ | @@ -4,9 +4,10 @@ | ||
4 | <link rel="import" href="../../bower_components/paper-input/paper-input.html"> | 4 | <link rel="import" href="../../bower_components/paper-input/paper-input.html"> |
5 | <link rel="import" href="../../bower_components/paper-icon-button/paper-icon-button.html"> | 5 | <link rel="import" href="../../bower_components/paper-icon-button/paper-icon-button.html"> |
6 | <link rel="import" href="../../bower_components/iron-icons/iron-icons.html"> | 6 | <link rel="import" href="../../bower_components/iron-icons/iron-icons.html"> |
7 | +<link rel="import" href="../../bower_components/iron-icons/maps-icons.html"> | ||
7 | <link rel="import" href="../../bower_components/paper-tooltip/paper-tooltip.html"> | 8 | <link rel="import" href="../../bower_components/paper-tooltip/paper-tooltip.html"> |
8 | 9 | ||
9 | -<script type="text/javascript" src="../../bower_components/JSDataChecker/jsdatachecker.min.js"></script> | 10 | +<script type="text/javascript" src="../../bower_components/jsdatachecker/jsdatachecker.min.js"></script> |
10 | 11 | ||
11 | <dom-module id="data-table-controllet"> | 12 | <dom-module id="data-table-controllet"> |
12 | 13 | ||
@@ -52,6 +53,10 @@ | @@ -52,6 +53,10 @@ | ||
52 | --paper-tooltip-background: black; | 53 | --paper-tooltip-background: black; |
53 | } | 54 | } |
54 | 55 | ||
56 | + iron-icon.icon { | ||
57 | + color: #2196F3; | ||
58 | + } | ||
59 | + | ||
55 | #data_table_container { | 60 | #data_table_container { |
56 | height: 100%; | 61 | height: 100%; |
57 | width: 100%; | 62 | width: 100%; |
@@ -205,7 +210,17 @@ | @@ -205,7 +210,17 @@ | ||
205 | <template is="dom-repeat" items="{{shownData}}"> | 210 | <template is="dom-repeat" items="{{shownData}}"> |
206 | <tr> | 211 | <tr> |
207 | <template is="dom-repeat" items="{{_toArray(item)}}"> | 212 | <template is="dom-repeat" items="{{_toArray(item)}}"> |
208 | - <td title="{{item.value}}">{{item.value}}</td> | 213 | + <template is="dom-if" if={{!_isObject(item)}}> |
214 | + <td title="{{item.value}}">{{item.value}}</td> | ||
215 | + </template> | ||
216 | + <template is="dom-if" if={{_isObject(item)}}> | ||
217 | + <template is="dom-if" if={{_isGEOJSON(item)}}> | ||
218 | + <td title="GEOJSON"><iron-icon icon="maps:map" class="icon"></iron-icon></td> | ||
219 | + </template> | ||
220 | + <template is="dom-if" if={{!_isGEOJSON(item)}}> | ||
221 | + <td title="OBJECT"><iron-icon icon=":help-outline" class="icon"></iron-icon></td> | ||
222 | + </template> | ||
223 | + </template> | ||
209 | </template> | 224 | </template> |
210 | </tr> | 225 | </tr> |
211 | </template> | 226 | </template> |
@@ -216,9 +231,9 @@ | @@ -216,9 +231,9 @@ | ||
216 | <paper-tooltip for="{{_fieldId(index)}}" offset="8"> | 231 | <paper-tooltip for="{{_fieldId(index)}}" offset="8"> |
217 | <p> | 232 | <p> |
218 | <span class="type">{{_type()}}:</span> {{item.type}} | 233 | <span class="type">{{_type()}}:</span> {{item.type}} |
219 | - <!--<template is="dom-if" if={{item.errorsDescription}}>--> | ||
220 | - <!--({{item.subType}})--> | ||
221 | - <!--</template>--> | 234 | + <template is="dom-if" if={{item.subtype}}> |
235 | + ({{item.subtype}}) | ||
236 | + </template> | ||
222 | <template is="dom-if" if={{item.errorsDescription.length}}> | 237 | <template is="dom-if" if={{item.errorsDescription.length}}> |
223 | <br/> | 238 | <br/> |
224 | <span class="warning">{{_warning()}}:</span> {{item.errorsDescription}} | 239 | <span class="warning">{{_warning()}}:</span> {{item.errorsDescription}} |
@@ -444,6 +459,19 @@ | @@ -444,6 +459,19 @@ | ||
444 | return "id_" + index; | 459 | return "id_" + index; |
445 | }, | 460 | }, |
446 | 461 | ||
462 | + _isObject: function(item) { | ||
463 | + if(item.value instanceof Array) | ||
464 | + return false; | ||
465 | + if(item.value instanceof Object) | ||
466 | + return true; | ||
467 | + return false; | ||
468 | + }, | ||
469 | + | ||
470 | + _isGEOJSON: function(item) { | ||
471 | + var dt = new DataTypeConverter(); | ||
472 | + return (dt.inferDataSubTypeOfValue(item.value) && dt.inferDataSubTypeOfValue(item.value).name == DataTypeConverter.SUBTYPES.GEOJSON.name); | ||
473 | + }, | ||
474 | + | ||
447 | _resize : function(){ | 475 | _resize : function(){ |
448 | this.async(function() { | 476 | this.async(function() { |
449 | var h = $(this.$.data_table_container).height() - 96; | 477 | var h = $(this.$.data_table_container).height() - 96; |
controllets/dataset-table-controllet/dataset-table-controllet.html
@@ -9,7 +9,7 @@ | @@ -9,7 +9,7 @@ | ||
9 | <link rel="import" href="../../bower_components/paper-dialog/paper-dialog.html"> | 9 | <link rel="import" href="../../bower_components/paper-dialog/paper-dialog.html"> |
10 | <link rel="import" href="../data-table-controllet/data-table-controllet.html"> | 10 | <link rel="import" href="../data-table-controllet/data-table-controllet.html"> |
11 | 11 | ||
12 | -<script type="text/javascript" src="../../bower_components/JSDataChecker/jsdatachecker.min.js"></script> | 12 | +<script type="text/javascript" src="../../bower_components/jsdatachecker/jsdatachecker.min.js"></script> |
13 | 13 | ||
14 | <dom-module id="dataset-table-controllet"> | 14 | <dom-module id="dataset-table-controllet"> |
15 | 15 |
controllets/providers-utility-controllet/openDataSoft.js
@@ -2,8 +2,11 @@ function openDataSoft_Provider () {} | @@ -2,8 +2,11 @@ function openDataSoft_Provider () {} | ||
2 | 2 | ||
3 | openDataSoft_Provider.prototype.selectData = function(data) { | 3 | openDataSoft_Provider.prototype.selectData = function(data) { |
4 | var selectedData = []; | 4 | var selectedData = []; |
5 | - for(var i=0; i<data.records.length; i++) | 5 | + for(var i=0; i<data.records.length; i++) { |
6 | + if(data.records[i].geometry) | ||
7 | + data.records[i].fields["geometry"] = data.records[i].geometry; | ||
6 | selectedData.push(data.records[i].fields); | 8 | selectedData.push(data.records[i].fields); |
9 | + } | ||
7 | return selectedData; | 10 | return selectedData; |
8 | }; | 11 | }; |
9 | 12 |
controllets/select-data-controllet/select-data-controllet.html
@@ -9,7 +9,7 @@ | @@ -9,7 +9,7 @@ | ||
9 | <script type="text/javascript" src="../../alasql-utility/alasql.min.js"></script> | 9 | <script type="text/javascript" src="../../alasql-utility/alasql.min.js"></script> |
10 | <script type="text/javascript" src="../../alasql-utility/alasql-utility.js"></script> | 10 | <script type="text/javascript" src="../../alasql-utility/alasql-utility.js"></script> |
11 | 11 | ||
12 | -<script type="text/javascript" src="../../bower_components/JSDataChecker/jsdatachecker.min.js"></script> | 12 | +<script type="text/javascript" src="../../bower_components/jsdatachecker/jsdatachecker.min.js"></script> |
13 | 13 | ||
14 | <dom-module id="select-data-controllet"> | 14 | <dom-module id="select-data-controllet"> |
15 | 15 | ||
@@ -144,8 +144,6 @@ | @@ -144,8 +144,6 @@ | ||
144 | setSelectedFields : function(selectedFields) { | 144 | setSelectedFields : function(selectedFields) { |
145 | this.async(function() { | 145 | this.async(function() { |
146 | var selectedIndices = utility_getSelectedIndices(this.fields, selectedFields); | 146 | var selectedIndices = utility_getSelectedIndices(this.fields, selectedFields); |
147 | - | ||
148 | - console.log(selectedIndices); | ||
149 | this.$.select_fields.setSelectFields(selectedIndices); | 147 | this.$.select_fields.setSelectFields(selectedIndices); |
150 | }, 0); | 148 | }, 0); |
151 | }, | 149 | }, |
controllets/select-visualization-controllet/select-visualization-controllet.html
@@ -358,7 +358,7 @@ | @@ -358,7 +358,7 @@ | ||
358 | var params = this.$.options.getParams(); | 358 | var params = this.$.options.getParams(); |
359 | for (var key in params) { this.params[key] = params[key]; } | 359 | for (var key in params) { this.params[key] = params[key]; } |
360 | 360 | ||
361 | - var data = alasql_complexSelectData(this.data, this.selectedFields, this.filters, inputs.getAggregators(), inputs.getOrders()); | 361 | + /*????? senza filtri"*/var data = alasql_complexSelectData(this.data, this.selectedFields, this.filters, inputs.getAggregators(), inputs.getOrders()); |
362 | data = transformData(data, this.selectedFields, true); | 362 | data = transformData(data, this.selectedFields, true); |
363 | 363 | ||
364 | //use cache | 364 | //use cache |
datalets/base-ajax-json-alasql-datalet/static/js/AjaxJsonAlasqlBehavior.js
@@ -124,7 +124,7 @@ var AjaxJsonAlasqlBehavior = { | @@ -124,7 +124,7 @@ var AjaxJsonAlasqlBehavior = { | ||
124 | result = converter.cast(result); | 124 | result = converter.cast(result); |
125 | data = result.dataset; | 125 | data = result.dataset; |
126 | 126 | ||
127 | - data = alasql_complexSelectData(data, fields, filters, aggregators, orders); | 127 | + data = alasql_complexSelectData(data, fields, [], aggregators, orders); |
128 | 128 | ||
129 | this.data = transformData(data, fields, true); | 129 | this.data = transformData(data, fields, true); |
130 | 130 |