Scripted Value Columns, Part V

“Make incremental progress; change comes not by the yard, but by the inch.”
Rick Pitino

Last time, we had enough parts cobbled together to demonstrate that the concept actually works. Of course, all we had to show for it was some random numbers, but that told us that the specified script was being called for each row, which is what we were after. Now that we know that the basic structure is performing as desired, we can revisit the configurable Script Include component and see if we can come up with some actual use cases that might be of value to someone.

One of the questions that triggered this idea was related to comments and work notes on Incidents. Assuming that the main record in the table is an Incident, we can clone our example Script Include to create one dedicated to pulling data out of the latest comment or work note on an Incident. We can call this new Script Include ScriptedJournalValueProvider.

var ScriptedJournalValueProvider = Class.create();
ScriptedJournalValueProvider.prototype = {
	initialize: function() {
	},

	getScriptedValue: function(item, config) {
		return Math.floor(Math.random() * 100) + '';
	},

	type: 'ScriptedJournalValueProvider'
};

We will want to delete the example code in the getScriptedValue function and come up with our own, but other than that, the basic structure remains the same. Assuming that we want our script to be able to handle a number of attributes of an Incident Journal entry, we can use the name of the column to determine which function will fetch us our value.

getScriptedValue: function(item, config) {
	var response = '';

	var column = config.name;
	if (column == 'last_comment') {
		response = this.getLastComment(item, config);
	} else if (column == 'last_comment_by') {
		response = this.getLastCommentBy(item, config);
	} else if (column == 'last_comment_on') {
		response = this.getLastCommentOn(item, config);
	} else if (column == 'last_comment_type') {
		response = this.getLastCommentType(item, config);
	}

	return response;
}

This way, we can point to this same script in multiple columns and the name of the column will determine which value from the last comment or work note gets returned.

Since all of the functions will need the data for the last entry, we should create a shared function that they all can leverage to obtain the record. As with many things on the ServiceNow platform, there are a number of ways to go about this, but for our demonstration purposes, we will read the sys_journal_field table looking for the last entry for the Incident in the current row.

getLastJournalEntry: function(sys_id) {
	var journalGR = new GlideRecord('sys_journal_field');
	journalGR.orderByDesc('sys_created_on');
	journalGR.addQuery('name', 'incident');
	journalGR.addQuery('element_id', sys_id);
	journalGR.setLimit(1);
	journalGR.query();
	journalGR.next();
	return journalGR;
}

Now that we have a common way to obtain the GlideRecord for the latest entry, we can start building our functions that extract the requested data value. Here is the one for the comment text.

getLastComment: function(item, config) {
	var response = '';

	var journalGR = this.getLastJournalEntry(item.sys_id);
	if (journalGR.isValidRecord()) {
		response = journalGR.getDisplayValue('value');
	}

	return response;
}

The others will basically be copies of the above, modified to return different values based on their purpose. The whole thing, all put together, now looks like this.

var ScriptedJournalValueProvider = Class.create();
ScriptedJournalValueProvider.prototype = {
	initialize: function() {
	},

	getScriptedValue: function(item, config) {
		var response = '';

		var column = config.name;
		if (column == 'last_comment') {
			response = this.getLastComment(item, config);
		} else if (column == 'last_comment_by') {
			response = this.getLastCommentBy(item, config);
		} else if (column == 'last_comment_on') {
			response = this.getLastCommentOn(item, config);
		} else if (column == 'last_comment_type') {
			response = this.getLastCommentType(item, config);
		}

		return response;
	},

	getLastComment: function(item, config) {
		var response = '';

		var journalGR = this.getLastJournalEntry(item.sys_id);
		if (journalGR.isValidRecord()) {
			response = journalGR.getDisplayValue('value');
		}

		return response;
	},

	getLastCommentBy: function(item, config) {
		var response = '';

		var journalGR = this.getLastJournalEntry(item.sys_id);
		if (journalGR.isValidRecord()) {
			response = journalGR.getDisplayValue('sys_created_by');
		}

		return response;
	},

	getLastCommentOn: function(item, config) {
		var response = '';

		var journalGR = this.getLastJournalEntry(item.sys_id);
		if (journalGR.isValidRecord()) {
			response = journalGR.getDisplayValue('sys_created_on');
		}

		return response;
	},

	getLastCommentType: function(item, config) {
		var response = '';

		var journalGR = this.getLastJournalEntry(item.sys_id);
		if (journalGR.isValidRecord()) {
			response = journalGR.getDisplayValue('element');
		}

		return response;
	},

	getLastJournalEntry: function(sys_id) {
		var journalGR = new GlideRecord('sys_journal_field');
		journalGR.orderByDesc('sys_created_on');
		journalGR.addQuery('name', 'incident');
		journalGR.addQuery('element_id', sys_id);
		journalGR.setLimit(1);
		journalGR.query();
		journalGR.next();
		return journalGR;
	},

	type: 'ScriptedJournalValueProvider'
};

Now that we have a Script Include to utilize, we need to put together a new page so that we can configure it to make use of it so that we can test it out. Let’s make a quick copy of the page that we were using for testing last time and call it scripted_value_test. Also, let’s make a quick copy of the test configuration script that we were using earlier and call it ScriptedValueConfig.

var ScriptedValueConfig = Class.create();
ScriptedValueConfig.prototype = Object.extendsObject(ContentSelectorConfig, {
	initialize: function() {
	},

	perspective: [{
		name: 'all',
		label: 'all',
		roles: ''
	}],

	state: [{
		name: 'all',
		label: 'All'
	}],

	table: {
		all: [{
			name: 'incident',
			displayName: 'Incident',
			all: {
				filter: 'caller_idDYNAMIC90d1921e5f510100a9ad2572f2b477fe^active=true',
				fields: 'number,opened_by,opened_at,short_description',
				svcarray: [{
					name: 'last_comment_on',
					label: 'Last Comment',
					heading: 'Last Comment',
					script: 'global.ScriptedJournalValueProvider'
				},{
					name: 'last_comment_by',
					label: 'Last Comment By',
					heading: 'Last Comment By',
					script: 'global.ScriptedJournalValueProvider'
				}],
				aggarray: [],
				btnarray: [],
				refmap: {
					sys_user: 'user_profile'
				},
				actarray: []
			}
		}]
	},

	type: 'ScriptedValueConfig'
});

Now let’s pull up our new page in the Service Portal Designer and point the table widget to our new configuration script.

Configuring the new test page to use the new test configuration script

Once we save that, we can pop over to the Service Portal and pull up our new page to try it out.

First test of our first real world utilization of this feature

Beautiful! Our new scripted value provider Script Include was called by the core SNH Data Table widget and it returned the requested values, which were then displayed on the list with all of the other standard table columns. That wasn’t so hard, now was it?

Of course, we still have a couple more wrapper widgets to modify (and test!), and I would like to produce another example, maybe something to do with catalog item variables, but I think we are close. One thing I see that I never noticed before, though, is that the added columns don’t quite line up with the original columns. Maybe it is a CSS thing, or maybe it is something a little more diabolical, but I want to take a look at that and see what is going on there. All of the data in the columns should be displayed consistently; I don’t like it when things don’t all line up correctly. I need to figure out what is going on there and see what I can do about it.

Anyway, we still have a little more work to do before we can wrap this all up into a new Update Set and post a new version out on Share, but we will keep plugging along in our next installment.

Collaboration Store, Part LXX

“Software bugs are like cockroaches; there are probably dozens hiding in difficult to reach places for every one you find and fix.”
Donald G. Firesmith

Last time, we went through the list of issues that have been reported so far, the biggest one being the fact that the REST API call to the Host instance is sending over the application logo image attachment instead of the Update Set XML file attachment. Since then, we have received some additional information in the form of the data logged to the REST API log file. Here is the entry of interest:

{
	“size_bytes”: “547670”,
	“file_name”: “logo”,
	“sys_mod_count”: “0”,
	“average_image_color”: “”,
	“image_width”: “”,
	“sys_updated_on”: “2022-08-02 16:55:55”,
	“sys_tags”: “”,
	“table_name”: “x_11556_col_store_member_application_version”,
	“sys_id”: “c227acc297855110b40ebde3f153aff3”,
	“image_height”: “”,
	“sys_updated_by”: “csworker1.dev69362”,
	“download_link”: “https://dev69362.service-now.com/api/now/attachment/c227acc297855110b40ebde3f153aff3/file”,
	“content_type”: “image/jpeg”,
	“sys_created_on”: “2022-08-02 16:55:55”,
	“size_compressed”: “247152”,
	“compressed”: “true”,
	“state”: “pending”,
	“table_sys_id”: “b127a88297855110b40ebde3f153afa6”,
	“chunk_size_bytes”: “700000”,
	“hash”: “8b5a07a6c0edf042df4b3c24e729036562985b705427ba7e33768566de94e96f”,
	“sys_created_by”: “csworker1.dev69362”
}

If you look at the table_name property, you can see that it is attaching something to the version record, and if you look at the file_name and content_type properties, you can see that it isn’t the Update Set XML file that it is sending over. So let’s take a look at the shared code that sends over the Update Set XML file attachment and see if we can see where things may have gone wrong.

pushAttachment: function(attachmentGR, targetGR, remoteVerId) {
	var result = {};

	var gsa = new GlideSysAttachment();
	result.url = 'https://';
	result.url += targetGR.getDisplayValue('instance');
	result.url += '.service-now.com/api/now/attachment/file?table_name=x_11556_col_store_member_application_version&table_sys_id=';
	result.url += remoteVerId;
	result.url += '&file_name=';
	result.url += attachmentGR.getDisplayValue('file_name');
	result.method = 'POST';
	var request = new sn_ws.RESTMessageV2();
	request.setEndpoint(result.url);
	request.setHttpMethod(result.method);
	request.setBasicAuth(this.WORKER_ROOT + targetGR.getDisplayValue('instance'), targetGR.getDisplayValue('token'));
	request.setRequestHeader('Content-Type', attachmentGR.getDisplayValue('content_type'));
	request.setRequestHeader('Accept', 'application/json');
	request.setRequestBody(gsa.getContent(attachmentGR));
	var response = request.execute();
	result.status = response.getStatusCode();
	result.body = response.getBody();
	if (result.body) {
		try {
			result.obj = JSON.parse(result.body);
		} catch (e) {
			result.parse_error = e.toString();
		}
	}
	result.error = response.haveError();
	if (result.error) {
		result.error_code = response.getErrorCode();
		result.error_message = response.getErrorMessage();
	}
	this.logRESTCall(targetGR, result);

	return result;
}

By this point in the process, the GlideRecord for the attachment has already been obtained from the database, so the problem has to be upstream from here. This is a shared function called from many places, but our problem is related to the application publishing process, so let’s take a look at the ApplicationPublisher Script Include and see if we can find where this function is called.

processPhase7: function(answer) {
	var gsa = new GlideSysAttachment();
	var attachmentGR = new GlideRecord('sys_attachment');
	if (attachmentGR.get(answer.attachmentId)) {
		var targetGR = this.getHostInstanceGR();
		var csu = new CollaborationStoreUtils();
		var result = csu.pushAttachment(attachmentGR, targetGR, answer.hostVerId);
		if (result.error) {
			answer = this.processError(answer, 'Error returned from Host instance: ' + result.error_code + ' - ' + result.error_message);
		} else if (result.parse_error) {
			answer = this.processError(answer, 'Unparsable JSON string returned from Host instance: ' + result.body);
		} else if (result.status != 200 && result.status != 201) {
			answer = this.processError(answer, 'Invalid HTTP Response Code returned from Host instance: ' + result.status);
		} else {
			answer.hostVerId = result.obj.result.sys_id;
		}
	} else {
		answer = this.processError(answer, 'Invalid attachment record sys_id: ' + answer.attachmentId);
	}

	return answer;
}

Here we are fetching the attachment record based on the sys_id in the answer object property called attachmentId. There isn’t much opportunity for things to go tango uniform with this particular code, so I think we have to assume that somewhere upstream of this logic the value of answer.attachmentId got set to the sys_id of the logo attachment instead of the sys_id of the Update Set XML file attachment. So it looks like we need to do a quick search for answer.attachmentId and see where this property may have gotten corrupted.

Since the version record does not yet exist when the Update Set XML file is generated, it is initially attached to the stock application record. Then, once the version record has been created, the attachment is copied from the application record to the version record, and then the original attachment file is removed from the stock application record. All of that seems to work, since the Update Set XML file is, in fact, attached to the version record on the original source instance; however, somewhere along the line, the sys_id of that attachment record in the answer object ends up being the sys_id of the logo image attachment record. Let’s take a look at that code.

processPhase4: function(answer) {
	var gsa = new GlideSysAttachment();
	var values = gsa.copy('sys_app', answer.appSysId, 'x_11556_col_store_member_application_version', answer.versionId);
	gsa.deleteAttachment(answer.attachmentId);
	if (values.length > 0) {
		var ids = values[values.length - 1].split(',');
		if (ids[1]) {
			answer.attachmentId = ids[1];
		} else {
			answer = this.processError(answer, 'Unrecognizable response from attachment copy: ' + JSON.stringify(values));
		}
	} else {
		answer = this.processError(answer, 'Unrecognizable response from attachment copy: ' +  JSON.stringify(values));
	}

	return answer;
}

This has to be the source of the problem. The copy method the GlideSysAttachment object doesn’t allow you to select what to copy; it arbitrarily copies all attachments from one record to another and returns an array of sys_id pairs (before and after for each attachment). The code above assumed that the last pair contained the sys_id that we were looking for, but apparently, that is not always the case. It looks like we need to examine every sys_id pair in the array, select the one that contains the XML file, grab that sys_id, and then delete all of the other attachments from the version record. That would mean replacing this:

var ids = values[values.length - 1].split(',');
if (ids[1]) {
	answer.attachmentId = ids[1];
}

… with this:

var origId = answer.attachmentId;
for (var i=0; i<values.length; i++) {
	var ids = values[i].split(',');
	if (ids[0] == origId) {
		answer.attachmentId = ids[1];
		gsa.deleteAttachment(origId);
	} else {
		gsa.deleteAttachment(ids[1]);
	}
}

Basically, this code loops through all of the sys_id pairs, looks for the one where the first sys_id matches the original, grabs the second sys_id of that pair for the new answer.attachmentId value, and then deletes the original attachment record. When the first sys_id does not match, then it deletes the copied attachment from the version record, as we did not want to copy that one anyway. We will have to do a little testing to prove this out, but hopefully this will resolve this issue.

Next time, we should have a new Update Set available with this, and a few other, minor corrections in it, and then we can do a little retesting and see if that resolves a few of these issues. As always, if anyone finds anything else that we need to address, please leave the details in the comments section below. All feedback is heartily welcomed!

Scripted Value Columns, Part IV

“Where you’re headed is more important than how fast you’re going.”
Stephen Covey

Last time, we modified the core SNH Data Table widget to process the new configuration properties for scripted value columns. Before we can try it out, though, we need to update one or more of the three wrapper widgets, since no one uses the core widget directly. Probably the simplest to take on would be the SNH Data Table from JSON Configuration widget, the one that was added to process a configuration script directly. As we did with the editor and the core data table widget, we can scan the code for the aggarray and then copy any code needed as a starting point for our modifications. The only reference that appears in this widget is in the Server script, and only on a single line:

data.aggarray = data.tableData.aggarray;

We can replicate that line, and then modify the copy for our new array of scripted value columns.

data.svcarray = data.tableData.svcarray;
data.aggarray = data.tableData.aggarray;

And that’s the only thing that needs to be done to update this widget to support the new feature. Now we can build a page to try things out and see if it all works. Or better yet, maybe there is already a page out there that we can use for this quick test. Down at the bottom of the widget form there is a list of pages that already use this widget. Maybe we can tinker with one of those, just to give this a quick look.

Related list of portal pages that include the SNH Data Table from JSON Configuration widget

The table_from_json page looks like a prime candidate. All we need to do is to pull it up in the Portal Page Designer, point the configuration script option to the script that we modified earlier, and then give the page a try.

First test of the new scripted value column using random numbers as values

So, there is the “test” column that we added, filled with random numbers from our new ScriptedValueExample Script Include. This test demonstrates that our modified wrapper widget successfully passed the data from our recently edited configuration script to the core data table widget, and the core data table widget successfully handled the new configuration option and obtained the value for the new column from the specified Script Include. Sweet! Now we need to come up with some real world examples of how this feature might be employed for specific purposes, and also update the remaining wrapper widgets to accommodate this new feature. That all sounds like a good topic for our next installment.

Collaboration Store, Part LXIX

“We all need people who will give us feedback. That’s how we improve.”
Bill Gates

Last time, we released a new batch of Update Sets for the latest iteration of this effort and put out a plea for folks to take it all out for a spin. We got quite a lot of good, detailed feedback this time (Thanks, Joe!), so let’s make a quick list of everything that has been reported so far.

  • Preview errors during install
  • Application publishing failed during logo image copy
  • Application publishing failed after logo image removal
  • Application publishing failed due to Host instance being off line
  • Application publishing succeeded with new logo image, but on Host instance, logo image was attached to the version record instead of the Update Set XML file

None of these are good, but let’s take a look at them one at a time.

Preview errors during install

This one, I am able to duplicate. I also received 20 Preview errors when installing the Update Set on a new instance. Every one of the errors is basically the same.

Preview errors from initial install

Every one of the 20 errors contains the same message text.

Could not find a record in sys_hub_flow_base for column model referenced in this update

Searching for that message, I came across this:

https://community.servicenow.com/community?id=community_question&sys_id=82095744db9c70d0fb1e0b55ca9619b2

The accepted answer seems to be that this error message comes out because the Flow that you are trying to install is not present on the target instance. Well, that’s understandable, since you haven’t committed the Update Set just yet, but it doesn’t seem to me that that should be considered an error. Everyone’s answer is just to accept the remote update, but if you are shooting for a clean install, it doesn’t really look good to have these errors pop up for no reason. I looked for a way to suppress them or eliminate them, but so far I have not found anything of value. So it looks like you just accept them and continue, which is what I suggested when I first put this out there to install, but I don’t really like it. Maybe one day I will find a way to keep these messages from coming out, but for now, this is just the way that it is.

Application publishing failed during logo image copy

This one I have not been able to duplicate, which is unfortunate, because I would like to resolve it, and resolve it in a way that I can prove by running tests before and after the fix. In all of my testing, I have never had an image copy fail, so I am not sure how to proceed. However, it does occur to me that a failed logo image copy should not kill the entire process. Yes, it would be good to have the image along with the rest of the artifacts, but if that is the only issue, it seems to me that the rest of the publishing process should proceed. Here is the copy image function as it stands in version 0.7:

copyLogoImage: function(answer) {
	var logoId = '';

	var gsa = new GlideSysAttachment();
	var values = gsa.copy('ZZ_YYsys_app', answer.appSysId, 'ZZ_YYx_11556_col_store_member_application', answer.mbrAppId);
	if (values.length > 0) {
		var ids = values[values.length - 1].split(',');
		if (ids[1]) {
			logoId = ids[1];
		} else {
			answer = this.processError(answer, 'Unrecognizable response from logo attachment copy: ' +  JSON.stringify(values));
		}
	} else {
		answer = this.processError(answer, 'Unrecognizable response from logo attachment copy: ' +  JSON.stringify(values));
	}

	return logoId;
}

The processError function that is called when things go South logs the details of the error, displays a message, and then adds an error property to the answer object. I think if I remove the error property from the answer object, then the publication process will not stop at this point and everything will continue as if there was no image associated with the application. This seems like the preferable approach, at least to me. Maybe something like this:

copyLogoImage: function(answer) {
	var logoId = '';

	var gsa = new GlideSysAttachment();
	var values = gsa.copy('ZZ_YYsys_app', answer.appSysId, 'ZZ_YYx_11556_col_store_member_application', answer.mbrAppId);
	if (values.length > 0) {
		var ids = values[values.length - 1].split(',');
		if (ids[1]) {
			logoId = ids[1];
		} else {
			answer = this.processError(answer, 'Unrecognizable response from logo attachment copy: ' +  JSON.stringify(values));
			delete answer.error;
		}
	} else {
		answer = this.processError(answer, 'Unrecognizable response from logo attachment copy: ' +  JSON.stringify(values));
		delete answer.error;
	}

	return logoId;
}

That still doesn’t explain why this particular image could not be copied, but at least it would allow the publishing of the application to continue.

Application publishing failed after logo image removal

This is another one that I cannot seem to duplicate. The code related to an application image is fairly straightforward: if the app has an image and the store record does not, then it copies it over; otherwise, it does not do anything at all. If the app had no image, then if the publishing failed, it must have failed somewhere else, as the image copy function should not have even been invoked. Here is the relevant section of code:

if (sysAppGR.getValue('logo') && !mbrAppGR.getValue('logo')) {
	mbrAppGR.setValue('logo', this.copyLogoImage(answer));
}

If the app had no logo image, then nothing should have happened. I will have to look into this one a little deeper any maybe ask for a little more information before I understand what happened on this one.

Application publishing failed due to Host instance being off line

This is not actually a problem with the app, as there is no way to publish an application to a Host that is not up and running. but it does bring up an interesting question: should we check to see if the Host is available before we launch the process? That would at least prevent someone from going through half of the process only to have it die when it tries to move the artifacts over to the Host. We already have a getStoreInfo function that would tell us if the Host was available or not, so it wouldn’t take much to add a quick check before we launched the publishing process, and then inform the operator if things were not going to work out.

Application publishing succeeded with new logo image, but on Host instance, logo image was attached to the version record instead of the Update Set XML file

I have not found the source of this one just yet, but it appears to me that one or more sys_id values got passed to the wrong function or written to the wrong variable. Since everything turned out OK on the original Client, but ended up in the wrong place on the Host, the problem has to be in the REST API calls made from the Client to the Host. There are three calls that move attachments, one for the instance logo image, one for the application logo image, and one for the Update Set XML file attached to the version record. Either the logo image API call attached the logo to the wrong base record or the Update Set XML file call sent over the wrong attachment. A review of the relevant REST API call log records might reveal which one caused the problem, but I will dig through the code for both and see if I can understand how this might have happened. Obviously, you cannot install the app if you don’t have the Update Set XML file attached to the version record. This one definitely has to be fixed.

This was all great feedback, and very detailed, including copies of log file entries. That is very helpful in diagnosing these issue. If anyone else is having similar issues, please report them as well, and include as much information as you feel would be appropriate. And if someone has pulled this down and was able to run things without running into these issues, I would love to hear about that as well. As always, all feedback is welcome, positive, negative, or otherwise.

And Joe, if you are still willing to do a little more testing, try to publish a different app from your other Client, and see if you run into any similar issues with that. If you can find a fourth instance to join your trio, you might have the owner of that instance give this a shot as well. And thanks again for your assistance. It is very much appreciated. Thanks to all of you for helping to make this work the way that it should. I look forward to hearing more from anyone willing to give this all a try. Next time, we will take a look at any additional feedback, as well as any modifications that have been implemented as a result of the feedback that we have received thus far.

Scripted Value Columns, Part III

“Every day you may make progress. Every step may be fruitful. Yet there will stretch out before you an ever-lengthening, ever-ascending, ever-improving path.”
Winston Churchill

Last time, we took care of the configuration script editor and now we need to turn our attention to the main SNH Data Table Widgets starting with the core widget, SNH Data Table. As we did with the editor, we can search the various sections of the widget for aggarray, copy the relevant code, and modify it to handle the new svcarrary. As usual, we can start with the HTML, where we find a couple of sections, one for the headings:

<th ng-repeat="scripted in data.svcarray" class="text-nowrap center" tabindex="0">
  {{scripted.heading || scripted.label}}
</th>

… and one for the data columns:

<td ng-repeat="obj in item.svcValue" role="cell" class="sp-list-cell" ng-class="{selected: item.selected}" tabindex="0">
  {{obj.value}}
</td>

That takes care of the HTML. Now we need to take a look at the Server script. The first thing that we come across is this added block of comments:

// Start: SNH Data Table enhancements
	 * data.bulkactions = the JSON string containing the bulk action specifications
	 * data.refpage = the JSON string containing the reference link specifications
	 * data.aggregates = the JSON string containing the aggregate column specifications
	 * data.buttons = the JSON string containing the button specifications
	 * data.actarray = the bulk actions specifications object
	 * data.refmap = the reference link specifications object
	 * data.aggarray = the array of aggregate column specifications
	 * data.btnarray = the array of button specifications
// End: SNH Data Table enhancements

So we will modify that to include two new properties for our new feature.

// Start: SNH Data Table enhancements
	 * data.bulkactions = the JSON string containing the bulk action specifications
	 * data.refpage = the JSON string containing the reference link specifications
	 * data.scripteds = the JSON string containing the scripted value column specifications
	 * data.aggregates = the JSON string containing the aggregate column specifications
	 * data.buttons = the JSON string containing the button specifications
	 * data.actarray = the bulk actions specifications object
	 * data.refmap = the reference link specifications object
	 * data.svcarray = the array of scripted value column specifications
	 * data.aggarray = the array of aggregate column specifications
	 * data.btnarray = the array of button specifications
// End: SNH Data Table enhancements

The next reference to aggarray is this added variable copy statement:

// Start: SNH Data Table enhancements
	optCopy(['table_name', 'aggregates', 'buttons', 'btns', 'refpage', 'bulkactions', 'aggarray', 'btnarray', 'refmap', 'actarray', 'field_list']);

	...
// End: SNH Data Table enhancements

So we will add our new variables to this list.

// Start: SNH Data Table enhancements
	optCopy(['table_name', 'scripteds', 'aggregates', 'buttons', 'btns', 'refpage', 'bulkactions', 'svcarray', 'aggarray', 'btnarray', 'refmap', 'actarray', 'field_list']);

	...
// End: SNH Data Table enhancements

Shortly after that, we come to this code that validates and initializes the aggarray value.

if (data.aggregates) {
	try {
		var aggregateinfo = JSON.parse(data.aggregates);
		if (Array.isArray(aggregateinfo)) {
			data.aggarray = aggregateinfo;
		} else if (typeof aggregateinfo == 'object') {
			data.aggarray = [];
			data.aggarray[0] = aggregateinfo;
		} else {
			gs.error('Invalid aggregates option in SNH Data Table widget: ' + data.aggregates);
			data.aggarray = [];
		}
	} catch (e) {
		gs.error('Unparsable aggregates option in SNH Data Table widget: ' + data.aggregates);
		data.aggarray = [];
	}
} else {
	if (!data.aggarray) {
		data.aggarray = [];
	}
}

So we can copy that, and add a section just like it for the new svcarray.

if (data.scripteds) {
	try {
		var scriptedinfo = JSON.parse(data.scripteds);
		if (Array.isArray(scriptedinfo)) {
			data.svcarray = scriptedinfo;
		} else if (typeof scriptedinfo == 'object') {
			data.svcarray = [];
			data.svcarray[0] = scriptedinfo;
		} else {
			gs.error('Invalid scripteds option in SNH Data Table widget: ' + data.scripteds);
			data.svcarray = [];
		}
	} catch (e) {
		gs.error('Unparsable scripteds option in SNH Data Table widget: ' + data.scripteds);
		data.svcarray = [];
	}
} else {
	if (!data.svcarray) {
		data.svcarray = [];
	}
}

The next reference that we find is the code that actually adds the values to the records. For the aggregate columns, that code looks like this:

record.aggValue = [];
if (data.aggarray.length > 0) {
	for (var j=0; j<data.aggarray.length; j++) {
		var config = data.aggarray[j];
		var sysId = record.sys_id;
		if (config.source) {
			sysId = gr.getValue(config.source);
		}
		record.aggValue.push(getAggregateValue(sysId, config));
	}
}

We can make a copy of this section as well, but since the scripted values do not require a source property, our new section will be even simpler.

record.svcValue = [];
if (data.svcarray.length > 0) {
	for (var j=0; j<data.svcarray.length; j++) {
		record.svcValue.push(getScriptedValue(record, data.svcarray[j]));
	}
}

Of course, now we have referenced a function that doesn’t yet exist, but that is in fact the next and last reference that we come across. The function for the aggregates looks like this:

// Start: SNH Data Table enhancements
	function getAggregateValue(sys_id, config) {
		var value = 0;
		var ga = new GlideAggregate(config.table);
		ga.addAggregate('COUNT');
		var query = config.field + '=' + sys_id;
		if (config.filter) {
			query += '^' + config.filter;
		}
		ga.addEncodedQuery(query);
		ga.query();
		if (ga.next()) {
			value = parseInt(ga.getAggregate('COUNT'));
		}
		var response = {value: value};
		if (config.hint || config.page_id) {
			response.name = config.name;
		}
		return response;
	}
// End: SNH Data Table enhancements

Here is where we have to do something completely different from the original that we are copying. For the aggregate columns, we are actually doing the query to count the related records. For our new purpose, we are just going to grab an instance of the specified script and call the function on the script to get the value. Since we will be calling the same script for every row, it would be better to fetch the instance of the script once and hang on to it so that the same instance could be used again and again. To support that, we can establish a map of instances and an instance of the Instantiator up near the top.

var instantiator = new Instantiator(this);
var scriptMap = {};

With that in place, we can add the following new function to support the new scripted value columns.

function getScriptedValue(record, config) {
	var response = {value: ''};
	var scriptName = config.script;
	if (scriptName) {
		if (scriptName.startsWith('global.')) {
			scriptName = scriptName.split('.')[1];
		}
		if (!scriptMap[scriptName]) {
			scriptMap[scriptName] = instantiator.getInstance(scriptName);
		}
		if (scriptMap[scriptName]) {
			response.value = scriptMap[scriptName].getScriptedValue(record, config);
		}
	}
	return response;
}

That’s it for the Server script. The whole thing now looks like this:

(function() {
	if (!input) // asynch load list
		return;

	data.msg = {};
	data.msg.sortingByAsc = gs.getMessage("Sorting by ascending");
	data.msg.sortingByDesc = gs.getMessage("Sorting by descending");

	/*
	 * data.table = the table
	 * data.p = the current page starting at 1
	 * data.o = the order by column
	 * data.d = the order by direction
	 * data.keywords = the keyword search term
	 * data.list = the table data as an array
	 * data.invalid_table = true if table is invalid or if data was not succesfully fetched
	 * data.table_label = the table's display name. e.g. Incident
	 * data.table_plural = the table's plural display name. e.g. Incidents
	 * data.fields = a comma delimited list of field names to show in the data table
	 * data.column_labels = a map of field name -> display name
	 * data.window_size = the number of rows to show
	 * data.filter = the encoded query
// Start: SNH Data Table enhancements
	 * data.bulkactions = the JSON string containing the bulk action specifications
	 * data.refpage = the JSON string containing the reference link specifications
	 * data.scripteds = the JSON string containing the scripted value column specifications
	 * data.aggregates = the JSON string containing the aggregate column specifications
	 * data.buttons = the JSON string containing the button specifications
	 * data.actarray = the bulk actions specifications object
	 * data.refmap = the reference link specifications object
	 * data.svcarray = the array of scripted value column specifications
	 * data.aggarray = the array of aggregate column specifications
	 * data.btnarray = the array of button specifications
// End: SNH Data Table enhancements
	 */
	// copy to data[name] from input[name] || option[name]
	optCopy(['table', 'p', 'o', 'd', 'filter', 'filterACLs', 'fields', 'keywords', 'view']);
	optCopy(['relationship_id', 'apply_to', 'apply_to_sys_id', 'window_size']);

// Start: SNH Data Table enhancements
	optCopy(['table_name', 'scripteds', 'aggregates', 'buttons', 'btns', 'refpage', 'bulkactions', 'svcarray', 'aggarray', 'btnarray', 'refmap', 'actarray', 'field_list']);

	// for some reason, 'buttons' and 'table' sometimes get lost in translation ...
	if (data.btns) {
		data.buttons = data.btns;
	}
	if (data.table_name) {
		data.table = data.table_name;
	}
// End: SNH Data Table enhancements

	if (!data.table) {
		data.invalid_table = true;
		data.table_label = "";
		return;
	}

// Start: SNH Data Table enhancements
	var instantiator = new Instantiator(this);
	var scriptMap = {};
	if (data.scripteds) {
		try {
			var scriptedinfo = JSON.parse(data.scripteds);
			if (Array.isArray(scriptedinfo)) {
				data.svcarray = scriptedinfo;
			} else if (typeof scriptedinfo == 'object') {
				data.svcarray = [];
				data.svcarray[0] = scriptedinfo;
			} else {
				gs.error('Invalid scripteds option in SNH Data Table widget: ' + data.scripteds);
				data.svcarray = [];
			}
		} catch (e) {
			gs.error('Unparsable scripteds option in SNH Data Table widget: ' + data.scripteds);
			data.svcarray = [];
		}
	} else {
		if (!data.svcarray) {
			data.svcarray = [];
		}
	}

	if (data.aggregates) {
		try {
			var aggregateinfo = JSON.parse(data.aggregates);
			if (Array.isArray(aggregateinfo)) {
				data.aggarray = aggregateinfo;
			} else if (typeof aggregateinfo == 'object') {
				data.aggarray = [];
				data.aggarray[0] = aggregateinfo;
			} else {
				gs.error('Invalid aggregates option in SNH Data Table widget: ' + data.aggregates);
				data.aggarray = [];
			}
		} catch (e) {
			gs.error('Unparsable aggregates option in SNH Data Table widget: ' + data.aggregates);
			data.aggarray = [];
		}
	} else {
		if (!data.aggarray) {
			data.aggarray = [];
		}
	}

	if (data.buttons) {
		try {
			var buttoninfo = JSON.parse(data.buttons);
			if (Array.isArray(buttoninfo)) {
				data.btnarray = buttoninfo;
			} else if (typeof buttoninfo == 'object') {
				data.btnarray = [];
				data.btnarray[0] = buttoninfo;
			} else {
				gs.error('Invalid buttons option in SNH Data Table widget: ' + data.buttons);
				data.btnarray = [];
			}
		} catch (e) {
			gs.error('Unparsable buttons option in SNH Data Table widget: ' + data.buttons);
			data.btnarray = [];
		}
	} else {
		if (!data.btnarray) {
			data.btnarray = [];
		}
	}

	if (data.refpage) {
		try {
			var refinfo = JSON.parse(data.refpage);
			if (typeof refinfo == 'object') {
				data.refmap = refinfo;
			} else {
				gs.error('Invalid reference page option in SNH Data Table widget: ' + data.refpage);
				data.refmap = {};
			}
		} catch (e) {
			gs.error('Unparsable reference page option in SNH Data Table widget: ' + data.refpage);
			data.refmap = {};
		}
	} else {
		if (!data.refmap) {
			data.refmap = {};
		}
	}

	if (data.bulkactions) {
		try {
			var actioninfo = JSON.parse(data.bulkactions);
			if (Array.isArray(actioninfo)) {
				data.actarray = actioninfo;
			} else if (typeof actioninfo == 'object') {
				data.actarray = [];
				data.actarray[0] = actioninfo;
			} else {
				gs.error('Invalid bulk actions in SNH Data Table widget: ' + data.bulkactions);
				data.actarray = [];
			}
		} catch (e) {
			gs.error('Unparsable bulk actions in SNH Data Table widget: ' + data.bulkactions);
			data.actarray = [];
		}
	} else {
		if (!data.actarray) {
			data.actarray = [];
		}
	}

	if (!data.fields) {
		if (data.field_list) {
			data.fields = data.field_list;
		} else if (data.view) {
			data.fields = $sp.getListColumns(data.table, data.view);
		} else {
			data.fields = $sp.getListColumns(data.table);
		}
	}
// End: SNH Data Table enhancements

	data.view = data.view || 'mobile';
	data.table = data.table || $sp.getValue('table');
	data.filter = data.filter || $sp.getValue('filter');
	data.keywords = data.keywords || $sp.getValue('keywords');
	data.p = data.p || $sp.getValue('p') || 1;
	data.p = parseInt(data.p);
	data.o = data.o || $sp.getValue('o') || $sp.getValue('order_by');
	data.d = data.d || $sp.getValue('d') || $sp.getValue('order_direction') || 'asc';
	data.useTinyUrl = gs.getProperty('glide.use_tiny_urls') === 'true';
	data.tinyUrlMinLength = gs.getProperty('glide.tiny_url_min_length');

// Start: SNH Data Table enhancements
	if (data.filter && data.filter.indexOf('{{sys_id}}')) {
		data.filter = data.filter.replace('{{sys_id}}', $sp.getParameter('sys_id'));
	}
// End: SNH Data Table enhancements


	var grForMetaData = new GlideRecord(data.table);

	if (input.setOrderUserPreferences) {
		// update User Preferences on a manual sort for UI consistency
		gs.getUser().savePreference(data.table + ".db.order", data.o);
		gs.getUser().savePreference(data.table + ".db.order.direction", data.d == "asc" ? "" : "DESC");
		data.setOrderUserPreferences = false;
	}
	// if no sort specified, find a default column for UI consistency
	if (!data.o)
		getOrderColumn();

	data.page_index = data.p - 1;
	data.show_new = data.show_new || options.show_new;
	var windowSize = data.window_size || $sp.getValue('maximum_entries') || 20;
	windowSize = parseInt(windowSize);
	if (isNaN(windowSize) || windowSize < 1)
		windowSize = 20;
	data.window_size = windowSize;

	var gr;
	// FilteredGlideRecord is not supported in scoped apps, so GlideRecordSecure will always be used in an application scope
	if (typeof FilteredGlideRecord != "undefined" && (gs.getProperty("glide.security.ui.filter") == "true" || grForMetaData.getAttribute("glide.security.ui.filter") != null)) {
		gr = new FilteredGlideRecord(data.table);
		gr.applyRowSecurity();
	} else
		gr = new GlideRecordSecure(data.table);
	if (!gr.isValid()) {
		data.invalid_table = true;
		data.table_label = data.table;
		return;
	}

	data.canCreate = gr.canCreate();
	data.newButtonUnsupported = data.table == "sys_attachment";
	data.table_label = gr.getLabel();
	data.table_plural = gr.getPlural();
	data.title = input.useInstanceTitle && input.headerTitle ? gs.getMessage(input.headerTitle) : data.table_plural;
	data.hasTextIndex = $sp.hasTextIndex(data.table);
	if (data.filter) {
		if (data.filterACLs)
			gr = $sp.addQueryString(gr, data.filter);
		else
			gr.addEncodedQuery(data.filter);
	}
	if (data.keywords) {
		gr.addQuery('123TEXTQUERY321', data.keywords);
		data.keywords = null;
	}

	data.filter = gr.getEncodedQuery();

	if (data.relationship_id) {
		var rel = GlideRelationship.get(data.relationship_id);
		var target = new GlideRecord(data.table);
		var applyTo = new GlideRecord(data.apply_to);
		applyTo.get("sys_id", data.apply_to_sys_id);
		rel.queryWith(applyTo, target); // put the relationship query into target
		data.exportQuery = target.getEncodedQuery();
		gr.addEncodedQuery(data.exportQuery); // get the query the relationship made for us
	}
	if (data.exportQuery)
		data.exportQuery += '^' + data.filter;
	else
		data.exportQuery = data.filter;
	data.exportQueryEncoded = encodeURIComponent(data.exportQuery);
	if (data.o){
		if (data.d == "asc")
			gr.orderBy(data.o);
		else
			gr.orderByDesc(data.o);
		if (gs.getProperty("glide.secondary.query.sysid") == "true")
			gr.orderBy("sys_id");
	}

	data.window_start = data.page_index * data.window_size;
	data.window_end = (data.page_index + 1) * data.window_size;
	gr.chooseWindow(data.window_start, data.window_end);
	gr.setCategory("service_portal_list");
	gr._query();

	data.row_count = gr.getRowCount();
	data.num_pages = Math.ceil(data.row_count / data.window_size);
	data.column_labels = {};
	data.column_types = {};
	data.fields_array = data.fields.split(',');

	// use GlideRecord to get field labels vs. GlideRecordSecure
	for (var i in data.fields_array) {
		var field = data.fields_array[i];
		var ge = grForMetaData.getElement(field);
		if (ge == null)
			continue;

		data.column_labels[field] = ge.getLabel();
		data.column_types[field] = ge.getED().getInternalType();
	}

	data.list = [];
	while (gr._next()) {
		var record = {};
		$sp.getRecordElements(record, gr, data.fields);
		if (typeof FilteredGlideRecord != "undefined" && gr instanceof FilteredGlideRecord) {
			// FilteredGlideRecord doesn't do field-level
			// security, so take care of that here
			for (var f in data.fields_array) {
				var fld = data.fields_array[f];
				if (!gr.isValidField(fld))
					continue;

				if (!gr[fld].canRead()) {
					record[fld].value = null;
					record[fld].display_value = null;
				}
			}
		}
		record.sys_id = gr.getValue('sys_id');

// Start: SNH Data Table enhancements
		for (var f in data.fields_array) {
			var fld = data.fields_array[f];
			if (record[fld].type == 'reference') {
				var refGr = gr;
				var refFld = fld;
				if (fld.indexOf('.') != -1) {
					var parts = fld.split('.');
					for (var x=0;x<parts.length-1;x++) {
						refGr = refGr[parts[x]].getRefRecord();
					}
					refFld = parts[parts.length-1];
				}
				if (refGr.isValidField(refFld)) {
					record[fld].table = refGr.getElement(refFld).getED().getReference();
					record[fld].record = {type: 'reference', sys_id: {value: record[fld].value, display_value: record[fld].value}, name: {value: record[fld].display_value, display_value: record[fld].display_value}};
				}
			}
		}
		record.svcValue = [];
		if (data.svcarray.length > 0) {
			for (var j=0; j<data.svcarray.length; j++) {
				record.svcValue.push(getScriptedValue(record, data.svcarray[j]));
			}
		}
		record.aggValue = [];
		if (data.aggarray.length > 0) {
			for (var k=0; k<data.aggarray.length; k++) {
				var config = data.aggarray[k];
				var sysId = record.sys_id;
				if (config.source) {
					sysId = gr.getValue(config.source);
				}
				record.aggValue.push(getAggregateValue(sysId, config));
			}
		}
// End: SNH Data Table enhancements

		record.targetTable = gr.getRecordClassName();
		data.list.push(record);
	}

	data.enable_filter = (input.enable_filter == true || input.enable_filter == "true" ||
		options.enable_filter == true || options.enable_filter == "true");
	var breadcrumbWidgetParams = {
		table: data.table,
		query: data.filter,
		enable_filter: data.enable_filter
	};
	data.filterBreadcrumbs = $sp.getWidget('widget-filter-breadcrumbs', breadcrumbWidgetParams);

	// copy to data from input or options
	function optCopy(names) {
		names.forEach(function(name) {
			data[name] = input[name] || options[name];
		})
	}

	// getOrderColumn logic mirrors that of Desktop UI when no sort column is specified
	function getOrderColumn() {
		// First check for user preference
		var pref = gs.getUser().getPreference(data.table + ".db.order");
		if (!GlideStringUtil.nil(pref)) {
			data.o = pref;
			if (gs.getUser().getPreference(data.table + ".db.order.direction") == "DESC")
				data.d = 'desc';
			return;
		}

		// If no user pref, check for table default using same logic as Desktop UI:
		// 1) if task, use number
		// 2) if any field has isOrder attribute, use that
		// 3) use order, number, name column if exists (in that priority)
		if (grForMetaData.isValidField("sys_id") && grForMetaData.getElement("sys_id").getED().getFirstTableName() == "task") {
			data.o = "number";
			return;
		}

		// Next check for isOrder attribute on any column
		var elements = grForMetaData.getElements();
		// Global and scoped GlideRecord.getElements return two different things,
		// so convert to Array if needed before looping through
		if (typeof elements.size != "undefined") {
			var elementArr = [];
			for (var i = 0; i < elements.size(); i++)
				elementArr.push(elements.get(i));
			elements = elementArr;
		}
		// Now we can loop through
		for (var j = 0; elements.length > j; j++) {
			var element = elements[j];
			if (element.getAttribute("isOrder") == "true") {
				data.o = element.getName();
				return;
			}
		}
		// As last resort, sort on Order, Number, or Name column
		if (grForMetaData.isValidField("order"))
			data.o = "order";
		else if (grForMetaData.isValidField("number"))
			data.o = "number";
		else if (grForMetaData.isValidField("name"))
			data.o = "name";
	}

// Start: SNH Data Table enhancements
	function getScriptedValue(record, config) {
		var response = {value: ''};
		var scriptName = config.script;
		if (scriptName) {
			if (scriptName.startsWith('global.')) {
				scriptName = scriptName.split('.')[1];
			}
			if (!scriptMap[scriptName]) {
				scriptMap[scriptName] = instantiator.getInstance(scriptName);
			}
			if (scriptMap[scriptName]) {
				response.value = scriptMap[scriptName].getScriptedValue(record, config);
			}
		}
		return response;
	}

	function getAggregateValue(sys_id, config) {
		var value = 0;
		var ga = new GlideAggregate(config.table);
		ga.addAggregate('COUNT');
		var query = config.field + '=' + sys_id;
		if (config.filter) {
			query += '^' + config.filter;
		}
		ga.addEncodedQuery(query);
		ga.query();
		if (ga.next()) {
			value = parseInt(ga.getAggregate('COUNT'));
		}
		var response = {value: value};
		if (config.hint || config.page_id) {
			response.name = config.name;
		}
		return response;
	}
// End: SNH Data Table enhancements

})();

There are no changes needed to the Client script, or any other area, so we are done with the modifications to this widget. Now would be a good time to try it out, but we will need at least one of the three wrapper widgets to be updated before we can give things a try. That sounds like a good project for our next installment.