Reusable Accessible Mapping Platform

API Docs for: 5.0.0
Show:

File: src\js\RAMP\Modules\dataLoader.js

  1. /* global define, console, Terraformer, shp, csv2geojson, RAMP, ArrayBuffer, Uint16Array */
  2.  
  3. /**
  4. * A module for loading from web services and local files. Fetches data via File API (IE9 is currently not supported) or
  5. * via XmlHttpRequest. Handles GeoJSON, Shapefiles and CSV currently. Includes utilities for parsing files into GeoJSON
  6. * (currently the selected intermediate format) and converting GeoJSON into FeatureLayers for consumption by the ESRI JS
  7. * API.
  8. *
  9. * @module RAMP
  10. * @submodule DataLoader
  11. * @uses dojo/Deferred
  12. * @uses dojo/query
  13. * @uses dojo/_base/array
  14. * @uses esri/request
  15. * @uses esri/SpatialReference
  16. * @uses esri/layers/FeatureLayer
  17. * @uses esri/renderers/SimpleRenderer
  18. * @uses ramp/layerLoader
  19. * @uses ramp/globalStorage
  20. * @uses ramp/map
  21. * @uses utils/util
  22. */
  23.  
  24. define([
  25. "dojo/Deferred", "dojo/query", "dojo/_base/array",
  26. "esri/request", "esri/SpatialReference", "esri/layers/FeatureLayer", "esri/renderers/SimpleRenderer",
  27. "ramp/layerLoader", "ramp/globalStorage", "ramp/map",
  28. "utils/util"
  29. ],
  30. function (
  31. Deferred, query, dojoArray,
  32. EsriRequest, SpatialReference, FeatureLayer, SimpleRenderer,
  33. LayerLoader, GlobalStorage, RampMap,
  34. Util
  35. ) {
  36. "use strict";
  37.  
  38. /**
  39. * Maps GeoJSON geometry types to a set of default renders defined in GlobalStorage.DefaultRenders
  40. * @property featureTypeToRenderer {Object}
  41. * @private
  42. */
  43. var featureTypeToRenderer = {
  44. Point: "circlePoint", MultiPoint: "circlePoint",
  45. LineString: "solidLine", MultiLineString: "solidLine",
  46. Polygon: "outlinedPoly", MultiPolygon: "outlinedPoly"
  47. };
  48.  
  49. /**
  50. * Loads a dataset using async calls, returns a promise which resolves with the dataset requested.
  51. * Datasets may be loaded from URLs or via the File API and depending on the options will be loaded
  52. * into a string or an ArrayBuffer.
  53. *
  54. * @param {Object} args Arguments object, should contain either {string} url or {File} file and optionally
  55. * {string} type as "text" or "binary" (text by default)
  56. * @returns {Promise} a Promise object resolving with either a {string} or {ArrayBuffer}
  57. */
  58. function loadDataSet(args) {
  59. var def = new Deferred(), promise;
  60.  
  61. if (args.file) {
  62. if (args.url) {
  63. throw new Error("Either url or file should be specified, not both");
  64. }
  65.  
  66. if (args.type === "binary") {
  67. promise = Util.readFileAsArrayBuffer(args.file);
  68. } else {
  69. promise = Util.readFileAsText(args.file);
  70. }
  71.  
  72. promise.then(function (data) { def.resolve(data); }, function (error) { def.reject(error); });
  73. } else if (args.url) {
  74. try {
  75. promise = (new EsriRequest({ url: args.url, handleAs: "text" })).promise;
  76. } catch (e) {
  77. def.reject(e);
  78. }
  79.  
  80. promise.then(
  81. function (data) {
  82. // http://updates.html5rocks.com/2012/06/How-to-convert-ArrayBuffer-to-and-from-String
  83. function str2ab(str) {
  84. var buf = new ArrayBuffer(str.length * 2), // 2 bytes for each char
  85. bufView = new Uint16Array(buf),
  86. i = 0, j = 0, strLen = str.length, code;
  87.  
  88. while (i < strLen) {
  89. // jshint bitwise:false
  90. code = str.charCodeAt(i++);
  91. if (code & 0xff00) {
  92. bufView[j++] = (0xff00 & code) >> 8;
  93. }
  94. bufView[j++] = 0xff & code;
  95. // jshint bitwise:true
  96. }
  97.  
  98. return buf.slice(0,j);
  99. }
  100.  
  101. if (args.type === 'binary') {
  102. def.resolve(str2ab(data));
  103. return;
  104. }
  105. def.resolve(data);
  106. },
  107. function (error) { def.reject(error); }
  108. );
  109. } else {
  110. throw new Error("One of url or file should be specified");
  111. }
  112.  
  113. return def.promise;
  114. }
  115.  
  116. /**
  117. * Fetch relevant data from a single feature layer endpoint. Returns a promise which
  118. * resolves with a partial list of properties extracted from the endpoint.
  119. *
  120. * @param {string} featureLayerEndpoint a URL pointing to an ESRI Feature Layer
  121. * @returns {Promise} a promise resolving with an object containing basic properties for the layer
  122. */
  123. function getFeatureLayer(featureLayerEndpoint) {
  124. var def = new Deferred(), promise;
  125.  
  126. try {
  127. promise = (new EsriRequest({ url: featureLayerEndpoint + '?f=json' })).promise;
  128. } catch (e) {
  129. def.reject(e);
  130. }
  131.  
  132. promise.then(
  133. function (data) {
  134. var res = {
  135. layerId: data.id, //TODO verifiy this. i think this is the index. we would want to use autoID?
  136. layerName: data.name,
  137. layerUrl: featureLayerEndpoint,
  138. geometryType: data.geometryType,
  139. fields: dojoArray.map(data.fields, function (x) { return x.name; }),
  140. renderer: data.drawingInfo.renderer
  141. };
  142.  
  143. def.resolve(res);
  144. },
  145. function (error) {
  146. console.log(error);
  147. def.reject(error);
  148. }
  149. );
  150.  
  151. return def.promise;
  152. }
  153.  
  154. /**
  155. * Fetch relevant data from a legend related to a feature layer endpoint. Returns a promise which
  156. * resolves with a partial list of properties extracted from the endpoint.
  157. *
  158. * @param {string} featureLayerEndpoint a URL pointing to an ESRI Feature Layer
  159. * @returns {Promise} a promise resolving with an object mapping legend labels to data URLs for those labels
  160. */
  161. function getFeatureLayerLegend(featureLayerEndpoint) {
  162. var def = new Deferred(), promise, legendUrl, idx, layerIdx;
  163.  
  164. //snip off last slash if there
  165. idx = featureLayerEndpoint.indexOf('/', featureLayerEndpoint.length - 1);
  166. if (idx > -1) {
  167. legendUrl = featureLayerEndpoint.substring(0, idx - 1);
  168. } else {
  169. legendUrl = featureLayerEndpoint;
  170. }
  171.  
  172. //snip off & store layer index, add legend to url
  173. idx = legendUrl.lastIndexOf('/');
  174. layerIdx = parseInt(legendUrl.substring(idx + 1));
  175. legendUrl = legendUrl.substring(0, idx) + '/legend?f=json';
  176.  
  177. try {
  178. promise = (new EsriRequest({ url: legendUrl })).promise;
  179. } catch (e) {
  180. def.reject(e);
  181. }
  182.  
  183. promise.then(
  184. function (data) {
  185. //find our layer in the legend
  186. var res = {};
  187. dojoArray.forEach(data.layers, function (layer) {
  188. if (layer.layerId === layerIdx) {
  189. dojoArray.forEach(layer.legend, function (legendItem) {
  190. res[legendItem.label] = "data:" + legendItem.contentType + ';base64,' + legendItem.imageData;
  191. });
  192. }
  193. });
  194.  
  195. def.resolve(res);
  196. },
  197. function (error) {
  198. console.log(error);
  199. def.reject(error);
  200. }
  201. );
  202.  
  203. return def.promise;
  204. }
  205.  
  206. /**
  207. * Fetch layer data from a WMS endpoint. This method will execute a WMS GetCapabilities
  208. * request against the specified URL, it requests WMS 1.3 and it is capable of parsing
  209. * 1.3 or 1.1.1 responses. It returns a promise which will resolve with basic layer
  210. * metadata and querying information.
  211. *
  212. * metadata response format:
  213. * { queryTypes: [mimeType], layers: [{name, desc, queryable(bool)}] }
  214. *
  215. * @param {string} wmsEndpoint a URL pointing to a WMS server (it must not include a query string)
  216. * @returns {Promise} a promise resolving with a metadata object (as specified above)
  217. */
  218. function getWmsLayerList(wmsEndpoint) {
  219. var def = new Deferred(), promise;
  220.  
  221. try {
  222. promise = (new EsriRequest({ url: wmsEndpoint + '?service=WMS&version=1.3&request=GetCapabilities', handleAs: 'xml' })).promise;
  223. } catch (e) {
  224. def.reject(e);
  225. }
  226.  
  227. // there might already be a way to do this in the parsing API
  228. // I don't know XML parsing well enough (and I don't want to)
  229. function getImmediateChild(node, childName) {
  230. var i;
  231. for (i = 0; i < node.childNodes.length; ++i) {
  232. if (node.childNodes[i].nodeName === childName) {
  233. return node.childNodes[i];
  234. }
  235. }
  236. return undefined;
  237. }
  238.  
  239. promise.then(
  240. function (data) {
  241. var layers, res = {};
  242.  
  243. try {
  244. layers = dojoArray.map(query('Layer > Name', data), function (nameNode) { return nameNode.parentNode; });
  245. res.layers = dojoArray.map(layers, function (x) {
  246. var name = getImmediateChild(x, 'Name').textContent,
  247. titleNode = getImmediateChild(x, 'Title');
  248. return {
  249. name: name,
  250. desc: titleNode ? titleNode.textContent : name,
  251. queryable: x.getAttribute('queryable') === '1'
  252. };
  253. });
  254. res.queryTypes = dojoArray.map(query('GetFeatureInfo > Format', data), function (node) { return node.textContent; });
  255. } catch (e) {
  256. def.reject(e);
  257. }
  258.  
  259. def.resolve(res);
  260. },
  261. function (error) {
  262. console.log(error);
  263. def.reject(error);
  264. }
  265. );
  266.  
  267. return def.promise;
  268. }
  269.  
  270. /**
  271. * Performs in place assignment of integer ids for a GeoJSON FeatureCollection.
  272. */
  273. function assignIds(geoJson) {
  274. if (geoJson.type !== 'FeatureCollection') {
  275. throw new Error("Assignment can only be performed on FeatureCollections");
  276. }
  277. dojoArray.forEach(geoJson.features, function (val, idx) {
  278. if (typeof val.id === "undefined") {
  279. val.id = idx;
  280. }
  281. });
  282. }
  283.  
  284. /**
  285. * Extracts fields from the first feature in the feature collection, does no
  286. * guesswork on property types and calls everything a string.
  287. */
  288. function extractFields(geoJson) {
  289. if (geoJson.features.length < 1) {
  290. throw new Error("Field extraction requires at least one feature");
  291. }
  292.  
  293. return dojoArray.map(Object.keys(geoJson.features[0].properties), function (prop) {
  294. return { name: prop, type: "esriFieldTypeString" };
  295. });
  296. }
  297.  
  298. /**
  299. * Will generate a generic datagrid config node for a set of layer attributes.
  300. *
  301. * @param {Array} fields an array of attribute fields for a layer
  302. * @returns {Object} an JSON config object for feature datagrid
  303. */
  304. function createDatagridConfig(fields) {
  305. function makeField(id, fn, wd, ttl, tp) {
  306. return {
  307. id: id,
  308. fieldName: fn,
  309. width: wd,
  310. orderable: false,
  311. type: 'string',
  312. alignment: 0,
  313. title: ttl,
  314. columnTemplate: tp
  315. };
  316. }
  317.  
  318. var dg = {
  319. rowsPerPage: 50,
  320. gridColumns: []
  321. };
  322.  
  323. dg.gridColumns.push(makeField('iconCol', '', '50px', 'Icon', 'graphic_icon'));
  324. dg.gridColumns.push(makeField('detailsCol', '', '60px', 'Details', 'details_button'));
  325.  
  326. dojoArray.forEach(fields, function (field, idx) {
  327. dg.gridColumns.push(makeField("col" + idx.toString(), field, '100px', field, 'title_span'));
  328. });
  329.  
  330. return dg;
  331. }
  332.  
  333. /**
  334. * Will generate a symbology config node for a ESRI feature service.
  335. * Uses the information from the feature layers renderer JSON definition
  336. *
  337. * @param {Object} renderer renderer object from feature layer endpoint
  338. * @param {Object} legendLookup object that maps legend label to data url of legend image
  339. * @returns {Object} an JSON config object for feature symbology
  340. */
  341. function createSymbologyConfig(renderer, legendLookup) {
  342. var symb = {
  343. type: renderer.type
  344. };
  345.  
  346. switch (symb.type) {
  347. case "simple":
  348. symb.label = renderer.label;
  349. symb.imageUrl = legendLookup[renderer.label];
  350.  
  351. break;
  352.  
  353. case "uniqueValue":
  354. if (renderer.defaultLabel) {
  355. symb.defaultImageUrl = legendLookup[renderer.defaultLabel];
  356. }
  357. symb.field1 = renderer.field1;
  358. symb.field2 = renderer.field2;
  359. symb.field3 = renderer.field3;
  360. symb.valueMaps = dojoArray.map(renderer.uniqueValueInfos, function (uvi) {
  361. return {
  362. label: uvi.label,
  363. value: uvi.value,
  364. imageUrl: legendLookup[uvi.label]
  365. };
  366. });
  367.  
  368. break;
  369. case "classBreaks":
  370. if (renderer.defaultLabel) {
  371. symb.defaultImageUrl = legendLookup[renderer.defaultLabel];
  372. }
  373. symb.field = renderer.field;
  374. symb.minValue = renderer.minValue;
  375. symb.rangeMaps = dojoArray.map(renderer.classBreakInfos, function (cbi) {
  376. return {
  377. label: cbi.label,
  378. maxValue: cbi.classMaxValue,
  379. imageUrl: legendLookup[cbi.label]
  380. };
  381. });
  382.  
  383. break;
  384. default:
  385. //Renderer we dont support
  386. console.log('encountered unsupported renderer type: ' + symb.type);
  387. //TODO make a stupid basic renderer to prevent things from breaking?
  388. }
  389.  
  390. return symb;
  391. }
  392.  
  393. /**
  394. * Peek at the CSV output (useful for checking headers).
  395. *
  396. * @param {string} data a string containing the CSV (or any DSV) data
  397. * @param {string} delimiter the delimiter used by the data, unlike other functions this will not guess a delimiter and
  398. * this parameter is required
  399. * @returns {Array} an array of arrays containing the parsed CSV
  400. */
  401. function csvPeek(data, delimiter) {
  402. return csv2geojson.dsv(delimiter).parseRows(data);
  403. }
  404.  
  405. /**
  406. * Converts a GeoJSON object into a FeatureLayer. Expects GeoJSON to be formed as a FeatureCollection
  407. * containing a uniform feature type (FeatureLayer type will be set according to the type of the first
  408. * feature entry). Accepts the following options:
  409. * - renderer: a string identifying one of the properties in defaultRenders
  410. * - sourceProjection: a string matching a proj4.defs projection to be used for the source data (overrides
  411. * geoJson.crs)
  412. * - targetWkid: an integer for an ESRI wkid, defaults to map wkid if not specified
  413. * - fields: an array of fields to be appended to the FeatureLayer layerDefinition (OBJECTID is set by default)
  414. *
  415. * @method makeGeoJsonLayer
  416. * @param {Object} geoJson An object following the GeoJSON specification, should be a FeatureCollection with
  417. * Features of only one type
  418. * @param {Object} opts An object for supplying additional parameters
  419. * @returns {FeatureLayer} An ESRI FeatureLayer
  420. */
  421. function makeGeoJsonLayer(geoJson, opts) {
  422. var esriJson, layerDefinition, layer, fs, targetWkid, srcProj,
  423. defaultRenderers = GlobalStorage.DefaultRenderers,
  424. layerID = LayerLoader.nextId();
  425.  
  426. layerDefinition = {
  427. objectIdField: "OBJECTID",
  428. fields: [{
  429. name: "OBJECTID",
  430. type: "esriFieldTypeOID"
  431. }]
  432. };
  433.  
  434. targetWkid = RAMP.map.spatialReference.wkid;
  435. assignIds(geoJson);
  436. layerDefinition.drawingInfo = defaultRenderers[featureTypeToRenderer[geoJson.features[0].geometry.type]];
  437.  
  438. if (opts) {
  439. if (opts.sourceProjection) {
  440. srcProj = opts.sourceProjection;
  441. }
  442. if (opts.targetWkid) {
  443. targetWkid = opts.targetWkid;
  444. }
  445. if (opts.fields) {
  446. layerDefinition.fields = layerDefinition.fields.concat(opts.fields);
  447. }
  448. }
  449.  
  450. if (layerDefinition.fields.length === 1) {
  451. layerDefinition.fields = layerDefinition.fields.concat(extractFields(geoJson));
  452. }
  453.  
  454. console.log('reprojecting ' + srcProj + ' -> EPSG:' + targetWkid);
  455. //console.log(geoJson);
  456. Terraformer.Proj.convert(geoJson, 'EPSG:' + targetWkid, srcProj);
  457. //console.log(geoJson);
  458. esriJson = Terraformer.ArcGIS.convert(geoJson, { sr: targetWkid });
  459. console.log('geojson -> esrijson converted');
  460. //console.log(esriJson);
  461. fs = { features: esriJson, geometryType: layerDefinition.drawingInfo.geometryType };
  462.  
  463. layer = new FeatureLayer({ layerDefinition: layerDefinition, featureSet: fs }, { mode: FeatureLayer.MODE_SNAPSHOT, id: layerID });
  464. // \(`O´)/ manually setting SR because it will come out as 4326
  465. layer.spatialReference = new SpatialReference({ wkid: targetWkid });
  466.  
  467. // TODO : refactor the hack
  468. // SZ_HACK
  469. layer.renderer._RAMP_rendererType = featureTypeToRenderer[geoJson.features[0].geometry.type];
  470.  
  471. //SZ TESTING -- this will be removed when the UI separates the layer creation an layer enhancement
  472. //enhanceFileFeatureLayer(layer, opts);
  473.  
  474. return layer;
  475. }
  476.  
  477. /**
  478. * Will take a feature layer built from user supplied data, and apply extra user options (such as symbology,
  479. * display field), and generate a config node for the layer. Accepts the following options:
  480. * - renderer: a string identifying one of the properties in defaultRenders
  481. * - color: color of the renderer
  482. * - icon: icon to display in grid and maptips
  483. * - nameField: descriptive name field for the layer
  484. * - datasetName: description of the name field
  485. *
  486. * @method enhanceFileFeatureLayer
  487. * @param {Object} featureLayer a feature layer object generated by makeGeoJsonLayer
  488. * @param {Object} opts An object for supplying additional parameters
  489. */
  490. function enhanceFileFeatureLayer(featureLayer, opts) {
  491. //make a minimal config object for this layer
  492. var newConfig = {
  493. id: featureLayer.id,
  494. displayName: opts.datasetName,
  495. nameField: opts.nameField,
  496. symbology: {
  497. type: "simple",
  498. imageUrl: opts.icon
  499. },
  500. datagrid: createDatagridConfig(opts.fields)
  501. },
  502. defaultRenderers = GlobalStorage.DefaultRenderers;
  503.  
  504. //backfill the rest of the config object with default values
  505. newConfig = GlobalStorage.applyFeatureDefaults(newConfig);
  506.  
  507. //add custom properties and event handlers to layer object
  508. RampMap.enhanceLayer(featureLayer, newConfig, true);
  509. featureLayer.ramp.type = GlobalStorage.layerType.feature; //TODO revisit
  510. featureLayer.ramp.load.state = "loaded"; //because we made the feature layer by hand, it already has it's layer definition, so it begins in loaded state. the load event never fires
  511. featureLayer.type = "Feature Layer"; //required to visible layer function
  512.  
  513. //plop config in global config object so everyone can access it.
  514. RAMP.config.layers.feature.push(newConfig);
  515.  
  516. //apply new renderer if one is defined
  517. if (opts.renderer && defaultRenderers.hasOwnProperty(opts.renderer)) {
  518. var rend = defaultRenderers[opts.renderer].renderer;
  519. if (opts.colour) {
  520. rend.symbol.color = opts.colour;
  521. }
  522.  
  523. featureLayer.renderer = new SimpleRenderer(rend);
  524. } else if (opts.colour) { // change only color of the renderer
  525. // SZ_HACK
  526. featureLayer.renderer.symbol.color = opts.colour;
  527. }
  528. }
  529.  
  530. /**
  531. * Constructs a FeatureLayer from CSV data.
  532. * @param {string} csvData the CSV data to be processed
  533. * @param {object} opts options to be set for the parser {string} latfield, {string} lonfield, {string} delimiter
  534. * @returns {Promise} a promise resolving with a {FeatureLayer}
  535. */
  536. function buildCsv(csvData, opts) {
  537. var def = new Deferred(), csvOpts = { latfield: 'Lat', lonfield: 'Long', delimiter: ',' };
  538.  
  539. if (opts) {
  540. if (opts.latfield) {
  541. csvOpts.latfield = opts.latfield;
  542. }
  543. if (opts.lonfield) {
  544. csvOpts.lonfield = opts.lonfield;
  545. }
  546. if (opts.delimiter) {
  547. csvOpts.delimiter = opts.delimiter;
  548. }
  549. }
  550.  
  551. try {
  552. csv2geojson.csv2geojson(csvData, csvOpts, function (err, data) {
  553. var jsonLayer;
  554.  
  555. if (err) {
  556. def.reject(err);
  557. console.log("conversion error");
  558. console.log(err);
  559. return;
  560. }
  561. console.log('csv parsed');
  562. console.log(data);
  563. jsonLayer = makeGeoJsonLayer(data, opts);
  564. def.resolve(jsonLayer);
  565. });
  566. } catch (e) {
  567. def.reject(e);
  568. }
  569.  
  570. return def.promise;
  571. }
  572.  
  573. /**
  574. * Constructs a FeatureLayer from a Shapefile.
  575. * @param {ArrayBuffer} shpData an ArrayBuffer of the Shapefile in zip format
  576. * @returns {Promise} a promise resolving with a {FeatureLayer}
  577. */
  578. function buildShapefile(shpData) {
  579. var def = new Deferred();
  580.  
  581. try {
  582. // window.crypto.subtle.digest({ name: "SHA-256" }, shpData).then(function (h) { var u8 = new Uint16Array(h); console.log(u8); });
  583. shp.getShapefile(shpData).then(function (geojson) {
  584. var jsonLayer;
  585. try {
  586. jsonLayer = makeGeoJsonLayer(geojson);
  587. def.resolve(jsonLayer);
  588. } catch (e) {
  589. def.reject(e);
  590. }
  591. }, function (error) {
  592. def.reject(error);
  593. });
  594. } catch (e) {
  595. def.reject(e);
  596. }
  597.  
  598. return def.promise;
  599. }
  600.  
  601. /**
  602. * Constructs a FeatureLayer from a GeoJSON string.
  603. * This wraps makeGeoJsonLayer in an async wrapper, this is unnecessary but provides a consistent API.
  604. * @param {string} jsonData a string containing the GeoJSON
  605. * @returns {Promise} a promise resolving with a {FeatureLayer}
  606. */
  607. function buildGeoJson(jsonData) {
  608. var def = new Deferred(), jsonLayer = null;
  609.  
  610. try {
  611. jsonLayer = makeGeoJsonLayer(JSON.parse(jsonData));
  612. def.resolve(jsonLayer);
  613. } catch (e) {
  614. def.reject(e);
  615. }
  616.  
  617. return def.promise;
  618. }
  619.  
  620. return {
  621. loadDataSet: loadDataSet,
  622. getFeatureLayer: getFeatureLayer,
  623. getFeatureLayerLegend: getFeatureLayerLegend,
  624. getWmsLayerList: getWmsLayerList,
  625. makeGeoJsonLayer: makeGeoJsonLayer,
  626. csvPeek: csvPeek,
  627. buildCsv: buildCsv,
  628. buildShapefile: buildShapefile,
  629. buildGeoJson: buildGeoJson,
  630. enhanceFileFeatureLayer: enhanceFileFeatureLayer,
  631. createDatagridConfig: createDatagridConfig,
  632. createSymbologyConfig: createSymbologyConfig
  633. };
  634. });