In my previous post I've shared a quick overview on the Azure Computer Vision API service and it's implementation. If you didn't read it yet, please do before proceeding to this reading!
With the basics and the CognitiveServices in place, let's move forward and create a custom image field that uses this service to handle the image cropping, tagging and alt text description, all with AI.
I'll be sharing the whole implementation in GitHub later and also a package plugin, but let's get into the implementation details first.
Custom Image Field
The first step is to create the custom field, for doing that, go to the core DB and duplicate the /sitecore/system/Field types/Simple Types/Image field item. Let's call it "AICroppedImage".
Keep everything as it is except the assembly and class fields
AICroppedImage Class
For the implementation, we just decompiled the code from Sitecore.Kernel (Sitecore.Shell.Applications.ContentEditor.Image) and made all our needed customizations.
1using Sitecore.Configuration;2using Sitecore.Data.Items;3using Sitecore.DependencyInjection;4using Sitecore.Diagnostics;5using Sitecore.Globalization;6using Sitecore.Resources.Media;7using Sitecore.Shell.Applications.ContentEditor;8using Sitecore.Web.UI.Sheer;9using System;10using System.IO;11using System.Text;12using System.Web;13using System.Web.UI;14using Microsoft.Extensions.DependencyInjection;15using System.Linq;16using Sitecore.Computer.Vision.CroppingImageField.Models.ImagesDetails;17using Sitecore.Computer.Vision.CroppingImageField.Services;18namespace Sitecore.Computer.Vision.CroppingImageField.Fields19{20 public class AICroppedImage : Image21 {22 private readonly string ThumbnailsId = Settings.GetSetting("Sitecore.Computer.Vision.CroppingImageField.AICroppingField.ThumbnailsFolderId");23 private readonly ICognitiveServices _cognitiveServices;24 private readonly ICroppingService _croppingService;2526 public AICroppedImage(ICognitiveServices cognitiveServices, ICroppingService croppingService) : base()27 {28 _cognitiveServices = cognitiveServices;29 _croppingService = croppingService;30 }31 public AICroppedImage() : base()32 {33 _cognitiveServices = ServiceLocator.ServiceProvider.GetService<ICognitiveServices>();34 _croppingService = ServiceLocator.ServiceProvider.GetService<ICroppingService>();35 }36 protected override void DoRender(HtmlTextWriter output)37 {38 Assert.ArgumentNotNull((object)output, nameof(output));39 Item mediaItem = this.GetMediaItem();40 string src;41 this.GetSrc(out src);42 string str1 = " src=\"" + src + "\"";43 string str2 = " id=\"" + this.ID + "_image\"";44 string str3 = " alt=\"" + (mediaItem != null ? HttpUtility.HtmlEncode(mediaItem["Alt"]) : string.Empty) + "\"";45 this.Attributes["placeholder"] = Translate.Text(this.Placeholder);46 string str = this.Password ? " type=\"password\"" : (this.Hidden ? " type=\"hidden\"" : "");47 this.SetWidthAndHeightStyle();48 output.Write("<input" + this.ControlAttributes + str + ">");49 this.RenderChildren(output);50 output.Write("<div id=\"" + this.ID + "_pane\" class=\"scContentControlImagePane\">");51 string clientEvent = Sitecore.Context.ClientPage.GetClientEvent(this.ID + ".Browse");52 output.Write("<div class=\"scContentControlImageImage\" onclick=\"" + clientEvent + "\">");53 output.Write("<iframe" + str2 + str1 + str3 + " frameborder=\"0\" marginwidth=\"0\" marginheight=\"0\" width=\"100%\" height=\"128\" " +54 "allowtransparency=\"allowtransparency\"></iframe>");55 output.Write("<div id=\"" + this.ID + "_thumbnails\">");56 output.Write(GetThumbnails());57 output.Write("</div>");58 output.Write("</div>");59 output.Write("<div>");60 output.Write("<div id=\"" + this.ID + "_details\" class=\"scContentControlImageDetails\">");61 string details = this.GetDetails();62 output.Write(details);63 output.Write("</div>");64 output.Write("</div>");65 }66 protected override void DoChange(Message message)67 {68 Assert.ArgumentNotNull((object)message, nameof(message));69 base.DoChange(message);70 if (Sitecore.Context.ClientPage.Modified)71 {72 this.Update();73 }74 if (string.IsNullOrEmpty(this.Value))75 {76 this.ClearImage();77 }78 SheerResponse.SetReturnValue(true);79 }80 protected new void BrowseImage(ClientPipelineArgs args)81 {82 Assert.ArgumentNotNull((object)args, nameof(args));83 base.BrowseImage(args);84 if (Sitecore.Context.ClientPage.Modified)85 {86 this.Update();87 }88 }89 protected new void ShowProperties(ClientPipelineArgs args)90 {91 Assert.ArgumentNotNull((object)args, nameof(args));92 base.ShowProperties(args);93 if (Sitecore.Context.ClientPage.Modified)94 {95 this.Update();96 }97 }98 public override void HandleMessage(Message message)99 {100 Assert.ArgumentNotNull((object)message, nameof(message));101 base.HandleMessage(message);102103 string name = message.Name;104 if (name == "contentimage:clear")105 {106 this.ClearImage();107 }108 else if (name == "contentimage:refresh")109 {110 this.Update();111 }112 }113 private void ClearImage()114 {115 if (this.Disabled)116 {117 return;118 }119 if (this.Value.Length > 0)120 {121 this.SetModified();122 }123 this.XmlValue = new XmlValue(string.Empty, "image");124 this.Value = string.Empty;125 this.Update();126 }127 protected new void Update()128 {129 string src;130 this.GetSrc(out src);131 SheerResponse.SetAttribute(this.ID + "_image", "src", src);132 SheerResponse.SetInnerHtml(this.ID + "_thumbnails", this.GetThumbnails());133 SheerResponse.SetInnerHtml(this.ID + "_details", this.GetDetails());134 SheerResponse.Eval("scContent.startValidators()");135 }136 private string GetDetails()137 {138 var empty = string.Empty;139 MediaItem mediaItem = this.GetMediaItem();140 if (mediaItem != null)141 {142 var innerItem = mediaItem.InnerItem;143 var stringBuilder = new StringBuilder();144 var xmlValue = this.XmlValue;145 stringBuilder.Append("<div>");146 var item = innerItem["Dimensions"];147 var str = HttpUtility.HtmlEncode(xmlValue.GetAttribute("width"));148 var str1 = HttpUtility.HtmlEncode(xmlValue.GetAttribute("height"));149 ImageDetails imageDetails;150 using (var streamReader = new MemoryStream())151 {152 var mediaStrm = mediaItem.GetMediaStream();153 mediaStrm.CopyTo(streamReader);154 imageDetails = _cognitiveServices.AnalyzeImage(streamReader.ToArray());155 }156 if (!string.IsNullOrEmpty(str) || !string.IsNullOrEmpty(str1))157 {158 var objArray = new object[] { str, str1, item };159 stringBuilder.Append(Translate.Text("Dimensions: {0} x {1} (Original: {2})", objArray));160 }161 else162 {163 var objArray1 = new object[] { item };164 stringBuilder.Append(Translate.Text("Dimensions: {0}", objArray1));165 }166 stringBuilder.Append("</div>");167 stringBuilder.Append("<div style=\"padding:2px 0px 0px 0px; text-align=left; \">");168 var str2 = HttpUtility.HtmlEncode(innerItem["Alt"]);169 var str3 = imageDetails.Description.Captions.FirstOrDefault()?.Text;170 if (!string.IsNullOrEmpty(str3) && !string.IsNullOrEmpty(str2))171 {172 var objArray2 = new object[] { str3, str2 };173 stringBuilder.Append(Translate.Text("AI Alternate Text: \"{0}\" (Default Alternate Text: \"{1}\")", objArray2));174 }175 else if (!string.IsNullOrEmpty(str3))176 {177 var objArray3 = new object[] { str3 };178 stringBuilder.Append(Translate.Text("AI Alternate Text: \"{0}\"", objArray3));179 }180 else181 {182 var objArray4 = new object[] { str2 };183 stringBuilder.Append(Translate.Text("Default Alternate Text: \"{0}\"", objArray4));184 }185 stringBuilder.Append("</br>");186 var objArray5 = new object[] { str3 };187 stringBuilder.Append(Translate.Text("Tags: \"{0}\"", string.Join(",", imageDetails.Description.Tags), objArray5));188 stringBuilder.Append("</div>");189 empty = stringBuilder.ToString();190 }191 if (empty.Length == 0)192 {193 empty = Translate.Text("This media item has no details.");194 }195 return empty;196 }197 private Item GetMediaItem()198 {199 var attribute = this.XmlValue.GetAttribute("mediaid");200 if (attribute.Length <= 0)201 {202 return null;203 }204 Language language = Language.Parse(this.ItemLanguage);205 return Sitecore.Client.ContentDatabase.GetItem(attribute, language);206 }207 private MediaItem GetSrc(out string src)208 {209 src = string.Empty;210 MediaItem mediaItem = (MediaItem)this.GetMediaItem();211 if (mediaItem == null)212 {213 return null;214 }215 var thumbnailOptions = MediaUrlOptions.GetThumbnailOptions(mediaItem);216 int result;217 if (!int.TryParse(mediaItem.InnerItem["Height"], out result))218 {219 result = 128;220 }221 thumbnailOptions.Height = Math.Min(128, result);222 thumbnailOptions.MaxWidth = 640;223 thumbnailOptions.UseDefaultIcon = true;224 src = MediaManager.GetMediaUrl(mediaItem, thumbnailOptions);225 return mediaItem;226 }227 private string GetThumbnails()228 {229 var html = new StringBuilder();230 var src = string.Empty;231 var mediaItem = this.GetSrc(out src);232 if (mediaItem == null)233 {234 return string.Empty;235 }236 html.Append("<ul id=" + this.ID + "_frame\" style=\"display: -ms-flexbox;display: flex;-ms-flex-direction: row;flex-direction: row;-ms-flex-wrap: wrap;flex-wrap: wrap;\">");237 var thumbnailFolderItem = Sitecore.Client.ContentDatabase.GetItem(new Sitecore.Data.ID(ThumbnailsId));238 if (thumbnailFolderItem != null && thumbnailFolderItem.HasChildren)239 {240 foreach (Item item in thumbnailFolderItem.Children)241 {242 GetThumbnailHtml(item, html, mediaItem);243 }244 }245246 html.Append("</ul>");247 return html.ToString();248 }249 private void GetThumbnailHtml(Item item, StringBuilder html, MediaItem mediaItem)250 {251 if (item.Fields["Size"]?.Value != null)252 {253 var values = item.Fields["Size"].Value.Split('x');254 var width = values[0];255 var height = values[1];256 int w, h;257 if (int.TryParse(width, out w) && Int32.TryParse(height, out h) && w > 0 && h > 0)258 {259 var imageSrc = _croppingService.GenerateThumbnailUrl(w, h, mediaItem);260 html.Append(string.Format("<li id=\"Frame_{0}_{1}\" style=\"width: {2}px; height: {3}px; position: relative; overflow: hidden; display: inline-block;border: solid 3px #fff;margin: 5px 5px 5px 0;\">" +261 "<img style=\"position: relative;position: absolute;left: 0;top: 0;margin: 0;display: block;width: auto; height: auto;min-width: 100%; min-height: 100%;max-height: none; max-width: none;\" " +262 "src=\"{4}\"><img /><span style=\"position: absolute;" +263 "top: 0;left: 0;padding: 2px 3px;background-color: #fff;opacity: 0.8;\">{5}</span></li>", this.ID, item.ID.ToShortID(), w, h, imageSrc, item.DisplayName));264 }265 }266 }267 }268}269
We're basically modifying the way Sitecore renders the field with some small customizations, basically to add the thumbnails generated by the Azure Cognitive service and also the Alt and Tags texts.
Ok, so that's very much it, let's deploy our code and see how it looks in the Sitecore Content Editor. The only thing you need to do next, is create a template and make use of the newly created "AI Cropped Image" field.
Et Voila! The image field is now rendering a few thumbnails that gives you an idea of the final results when rendering the image in the front-end. As you can see, it gives also some tags and a description ("Diego Maradona holding a ball") used as alt text, everything coming from the Azure AI service, awesome!
Make the field rendered to work as an OOTB Sitecore image field
Next step, is to make sure we can still using the Sitecore helpers for rendering this field. For making this possible, we want to customize the Sitecore.Pipelines.RenderField.GetImageFieldValue processor. Same as before, we decompile the OOTB code from Sitecore.Kernel and we make our updates there. Then just patch the config like that:
<pipelines>
<renderField>
<processor patch:after="processor[@type='Sitecore.Pipelines.RenderField.GetImageFieldValue, Sitecore.Kernel']"
type="Foundation.Vision.Pipelines.RenderAICroppingImageField, Foundation.Vision">
</processor>
</renderField>
</pipelines>
Here, we just need to add the newly created field type (AI Cropped Image) as a valid image field type by overriding the IsImage() method.
using Sitecore.Diagnostics;
using Sitecore.Pipelines.RenderField;
namespace Sitecore.Computer.Vision.CroppingImageField.Pipelines
{
public class RenderAICroppingImageField : GetImageFieldValue
{
public override void Process(RenderFieldArgs args)
{
Assert.ArgumentNotNull((object)args, nameof(args));
if (!this.IsImage(args))
{
return;
}
var renderer = this.CreateRenderer();
this.ConfigureRenderer(args, renderer);
this.SetRenderFieldResult(renderer.Render(), args);
}
protected override bool IsImage(RenderFieldArgs args)
{
return args.FieldTypeKey == "AI Cropped Image";
}
}
}
Make it working with GlassMapper
Now, we can do some quick updates to GlassMapper as well so we can benefit from the glass helpers. Let's add a custom field mapper, again after decompiling Glass.Mapper.Sc.DataMappers.SitecoreFieldImageMapper, we can just extend it to work in the same way with the newly introduced AI Cropping Image field.
using Glass.Mapper.Sc;
using Glass.Mapper.Sc.Configuration;
using Glass.Mapper.Sc.DataMappers;
using Sitecore.Data;
using Sitecore.Data.Fields;
using Sitecore.Data.Items;
using System;
using Sitecore.Computer.Vision.CroppingImageField.Fields;
namespace Sitecore.Computer.Vision.CroppingImageField.Mappers
{
public class AICroppedImageFieldMapper : AbstractSitecoreFieldMapper
{
public AICroppedImageFieldMapper(): base(typeof(AICroppedImage))
{
}
public override object GetField(Field field, SitecoreFieldConfiguration config, SitecoreDataMappingContext context)
{
var img = new AICroppedImage();
var sitecoreImage = new AICroppedImageField(field);
SitecoreFieldImageMapper.MapToImage(img, sitecoreImage);
return img;
}
public override void SetField(Field field, object value, SitecoreFieldConfiguration config, SitecoreDataMappingContext context)
{
var img = value as AICroppedImage;
if (field == null || img == null)
{
return;
}
var item = field.Item;
var sitecoreImage = new AICroppedImageField(field);
SitecoreFieldImageMapper.MapToField(sitecoreImage, img, item);
}
public override string SetFieldValue(object value, SitecoreFieldConfiguration config, SitecoreDataMappingContext context)
{
throw new NotImplementedException();
}
public override object GetFieldValue(string fieldValue, SitecoreFieldConfiguration config, SitecoreDataMappingContext context)
{
var item = context.Service.Database.GetItem(new ID(fieldValue));
if (item == null)
{
return null;
}
var imageItem = new MediaItem(item);
var image = new AICroppedImage();
SitecoreFieldImageMapper.MapToImage(image, imageItem);
return image;
}
}
}
We need also to create our custom field that inherits from Glass.Mapper.Sc.Fields.Image
using Glass.Mapper.Sc.Fields;
namespace Sitecore.Computer.Vision.CroppingImageField.Mappers
{
public class AICroppedImage : Image
{
}
}
Last step is to add the mapper to the create resolver from the GlassMapperSCCustom.cs
public static class GlassMapperScCustom
{
public static IDependencyResolver CreateResolver(){
var config = new Glass.Mapper.Sc.Config();
var dependencyResolver = new DependencyResolver(config);
// add any changes to the standard resolver here
dependencyResolver.DataMapperFactory.First(() => new AICroppedImageFieldMapper());
dependencyResolver.Finalise();
return dependencyResolver;
}
}
Custom Caching
In order to reduce the calls to the service, an extra layer of caching has been implemented. This cache, as any other Sitecore cache gets flushed after a publishing and the size can be easily configured through it's configuration.
ComputerVision[CroppedImages] custom cache
...
...
In my next post, I'll be sharing the front end implementation, the full media request flow and the customizations needed to make it working in your site. Stay tuned!
