Using CSOM with SAML Authentication (ADFS)

Topics: ADFS 3.0, SharePoint 2013, claims authentication, on-premise, Azure, CSOM, SAML

Description

Recently, I was tasked with making CSOM work with these SAML-enabled web applications and host-named site collections.  So I went to the great Google and Bing parts bins, found some things that I could build upon, and got to work.

My first iteration or attempt at making CSOM work with SAML was straightforward.  I found a couple of articles that described how to use a WebClient to pop the ADFS login dialog, which then set the security token that the CSOM ClientContext used.  Works great and it is not complicated.

Problem

But at about 3 a.m. one morning, I woke up with a thought: what if I wanted to do the same thing without prompting anybody, such as for running CSOM code as a scheduled task?  So I spent some evenings on the couch researching this, gathering useful “parts” from folks online, and writing additional code.  And Fiddler and Postman were my good friends, too.

The end result shown below is a functional console application that authenticates to ADFS, obtains a security token, extracts and repackages the FedAuth cookie, and uses the FedAuth cookie for the SharePoint CSOM ClientContext to do work against a SharePoint list.

In order to make this work, an intimate understanding of the ADFS configuration is needed.

Solution

Credit and thanks for certain sections of code go to the posts identified in the source code comments and bolded.  No wheels were reinvented in the making of this solution, only custom code.  Please visit those links highlighted in the source code below before examining this code.

Note that this code has a dependency on these NuGet packages, one of which delivers TokenHelper.cs and SharePointContext.cs.

<packages>
 <package id="AppForSharePointOnlineWebToolkit" version="3.1.2" targetFramework="net45" />
 <package id="Microsoft.AspNet.WebApi.Client" version="5.2.3" targetFramework="net45" />
 <package id="Microsoft.AspNet.WebApi.Core" version="5.2.3" targetFramework="net45" />
 <package id="Newtonsoft.Json" version="6.0.4" targetFramework="net45" />
</packages>

using System;
using System.IdentityModel.Protocols.WSTrust;
using System.IdentityModel.Tokens;
using System.IO;
using System.Net;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.ServiceModel;
using System.ServiceModel.Security;
using System.Text;
using System.Web;
using System.Xml;
using SP = Microsoft.SharePoint.Client;

namespace MyConsoleApp
{
 class Program
 {
 private static string adfsEndpoint = "https://somesite.xxxx.loc/_trust/"; // Endpoint for site from where we get the token 
 private static string adfsRealm = "urn:somesite.xxxx.loc:somesite"; // ADFS realm that contains the endpoint above
 private static CookieContainer fedAuth = new CookieContainer();
 private static string webUrl = "https://somesite.xxxx.loc/finance";
 private static string fedAuthRequestCtx = HttpUtility.UrlEncode(string.Format("{0}/_layouts/15/Authenticate.aspx?Source=%2ffinance", webUrl));

 static void Main(string[] args)
 {
 // BEGIN adapted for C# from https://blogs.msdn.microsoft.com/besidethepoint/2012/10/17/request-adfs-security-token-with-powershell/

 string adfsBaseUri = "https://sts.xxxx.loc"; // your ADFS farm URL
 string adfsTrustEndpoint = "usernamemixed"; // OOTB ADFS, do not change
 string adfsTrustPath = "adfs/services/trust/13"; // OOTB ADFS, do not change
 string domain = "EUROPE"; // domain of AD account; should be in config file or propmted from the command line
 string pwd = "Quercus9%%"; // password of domain account; should be a secure string from config file, but you get the idea
 string userName = "jdoe@xxxx.loc:"; // UPN of domain account in AD

 EndpointAddress ep = new EndpointAddress(adfsBaseUri + "/" + adfsTrustPath + "/" + adfsTrustEndpoint);
 WS2007HttpBinding binding = new WS2007HttpBinding(SecurityMode.TransportWithMessageCredential);
 binding.Security.Message.EstablishSecurityContext = false;
 binding.Security.Message.ClientCredentialType = MessageCredentialType.UserName;
 binding.Security.Transport.ClientCredentialType = HttpClientCredentialType.None;

 WSTrustChannelFactory wstcf = new WSTrustChannelFactory(binding, ep);
 wstcf.TrustVersion = TrustVersion.WSTrust13;
 NetworkCredential cred = new NetworkCredential(userName, pwd, domain);
 wstcf.Credentials.Windows.ClientCredential = cred;
 wstcf.Credentials.UserName.UserName = cred.UserName;
 wstcf.Credentials.UserName.Password = cred.Password;
 var channel = wstcf.CreateChannel();

 string[] tokenType = { "urn:oasis:names:tc:SAML:1.0:assertion", "urn:oasis:names:tc:SAML:2.0:assertion" };
 RequestSecurityToken rst = new RequestSecurityToken();
 rst.RequestType = RequestTypes.Issue;
 rst.AppliesTo = new EndpointReference(adfsRealm);
 rst.KeyType = KeyTypes.Bearer;
 rst.TokenType = tokenType[0]; // Use the first one because that is what SharePoint itself uses (as observed in Fiddler).

 RequestSecurityTokenResponse rstr = new RequestSecurityTokenResponse();
 var cbk = new System.Net.Security.RemoteCertificateValidationCallback(ValidateRemoteCertificate);
 ServicePointManager.ServerCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => { return true; };
 SecurityToken token = null;
 try
 {
 token = channel.Issue(rst, out rstr);
 }
 finally
 {
 ServicePointManager.ServerCertificateValidationCallback = cbk;
 }

 // END adapted for C# from https://blogs.msdn.microsoft.com/besidethepoint/2012/10/17/request-adfs-security-token-with-powershell/

 // BEGIN adapted from https://github.com/OfficeDev/PnP-Sites-Core/blob/master/Core/SAML%20authentication.md

 Cookie fedAuthCookie = TransformTokenToFedAuth(((GenericXmlSecurityToken)token).TokenXml.OuterXml);
 fedAuth.Add(fedAuthCookie);

 // END adapted from https://github.com/OfficeDev/PnP-Sites-Core/blob/master/Core/SAML%20authentication.md

 // PERFORM CSOM OPS HERE:

 string listName = "Some Existing List";
 using (SP.ClientContext ctx = new SP.ClientContext(webUrl))
 {
 ctx.ExecutingWebRequest += ctx_ExecutingWebRequest;
 SP.CamlQuery query1 = new SP.CamlQuery();
 query1.ViewXml =
 "<View>" +
 "<Query>" +
 "<OrderBy>" +
 "<FieldRef Name='Title' Ascending='FALSE'/>" +
 "</OrderBy>" +
 "</Query>" +
 "<RowLimit>10</RowLimit>" +
 "</View>";
 SP.List list = ctx.Web.Lists.GetByTitle(listName);
 SP.ListItemCollection items = list.GetItems(query1);
 ctx.Load(items);
 ctx.ExecuteQuery();
 for (int i = 0; i < items.Count; i++)
 {
 SP.ListItem item = items[i];
 ctx.Load(item);
 ctx.ExecuteQuery();
 }
 }
 }

 static void ctx_ExecutingWebRequest(object sender, SP.WebRequestEventArgs e)
 {
 e.WebRequestExecutor.WebRequest.CookieContainer = fedAuth;
 }

 private static Cookie TransformTokenToFedAuth(string samlToken)
 {
 // Adapted from https://github.com/OfficeDev/PnP-Sites-Core/blob/master/Core/OfficeDevPnP.Core/IdentityModel/TokenProviders/ADFS/BaseProvider.cs

 samlToken = WrapInSoapMessage(samlToken, adfsRealm);
 string stringData = String.Format("wa=wsignin1.0&wctx={0}&wresult={1}", fedAuthRequestCtx, HttpUtility.UrlEncode(samlToken));
 HttpWebRequest req = HttpWebRequest.Create(adfsEndpoint) as HttpWebRequest;
 req.Method = "POST";
 req.ContentType = "application/x-www-form-urlencoded";
 req.CookieContainer = new CookieContainer();
 req.AllowAutoRedirect = false;
 Stream newStream = req.GetRequestStream();

 byte[] data = Encoding.UTF8.GetBytes(stringData);
 newStream.Write(data, 0, data.Length);
 newStream.Close();
 HttpWebResponse resp = req.GetResponse() as HttpWebResponse;
 var encoding = ASCIIEncoding.ASCII;
 string responseText = "";
 using (var reader = new System.IO.StreamReader(resp.GetResponseStream(), encoding))
 {
 responseText = reader.ReadToEnd();
 }
 return resp.Cookies["FedAuth"];
 }

 [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Globalization", "CA1303:Do not pass literals as localized parameters", MessageId = "System.Xml.XmlDocument.CreateTextNode(System.String)")]
 private static string WrapInSoapMessage(string stsResponse, string relyingPartyIdentifier)
 {
 // This method code is unchanged in its entirety from https://github.com/OfficeDev/PnP-Sites-Core/blob/master/Core/OfficeDevPnP.Core/IdentityModel/TokenProviders/ADFS/BaseProvider.cs.
 // I added in two places some inline commented code, which deals with using SAML 2.0 XML schema instead of 1.x. But SharePoint doesn't use 2.0. 

 XmlDocument samlAssertion = new XmlDocument();
 samlAssertion.PreserveWhitespace = true;
 samlAssertion.LoadXml(stsResponse);

 //Select the book node with the matching attribute value.
 String notBefore = /*samlAssertion.DocumentElement["Conditions"]*/samlAssertion.DocumentElement.FirstChild.Attributes["NotBefore"].Value;
 String notOnOrAfter = /*samlAssertion.DocumentElement["Conditions"]*/samlAssertion.DocumentElement.FirstChild.Attributes["NotOnOrAfter"].Value;

 XmlDocument soapMessage = new XmlDocument();
 XmlElement soapEnvelope = soapMessage.CreateElement("t", "RequestSecurityTokenResponse", "http://schemas.xmlsoap.org/ws/2005/02/trust");
 soapMessage.AppendChild(soapEnvelope);
 XmlElement lifeTime = soapMessage.CreateElement("t", "Lifetime", soapMessage.DocumentElement.NamespaceURI);
 soapEnvelope.AppendChild(lifeTime);
 XmlElement created = soapMessage.CreateElement("wsu", "Created", "http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd");
 XmlText createdValue = soapMessage.CreateTextNode(notBefore);
 created.AppendChild(createdValue);
 lifeTime.AppendChild(created);
 XmlElement expires = soapMessage.CreateElement("wsu", "Expires", "http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd");
 XmlText expiresValue = soapMessage.CreateTextNode(notOnOrAfter);
 expires.AppendChild(expiresValue);
 lifeTime.AppendChild(expires);
 XmlElement appliesTo = soapMessage.CreateElement("wsp", "AppliesTo", "http://schemas.xmlsoap.org/ws/2004/09/policy");
 soapEnvelope.AppendChild(appliesTo);
 XmlElement endPointReference = soapMessage.CreateElement("wsa", "EndpointReference", "http://www.w3.org/2005/08/addressing");
 appliesTo.AppendChild(endPointReference);
 XmlElement address = soapMessage.CreateElement("wsa", "Address", endPointReference.NamespaceURI);
 XmlText addressValue = soapMessage.CreateTextNode(relyingPartyIdentifier);
 address.AppendChild(addressValue);
 endPointReference.AppendChild(address);
 XmlElement requestedSecurityToken = soapMessage.CreateElement("t", "RequestedSecurityToken", soapMessage.DocumentElement.NamespaceURI);
 XmlNode samlToken = soapMessage.ImportNode(samlAssertion.DocumentElement, true);
 requestedSecurityToken.AppendChild(samlToken);
 soapEnvelope.AppendChild(requestedSecurityToken);
 XmlElement tokenType = soapMessage.CreateElement("t", "TokenType", soapMessage.DocumentElement.NamespaceURI);
 XmlText tokenTypeValue = soapMessage.CreateTextNode("urn:oasis:names:tc:SAML:1.0:assertion");
 tokenType.AppendChild(tokenTypeValue);
 soapEnvelope.AppendChild(tokenType);
 XmlElement requestType = soapMessage.CreateElement("t", "RequestType", soapMessage.DocumentElement.NamespaceURI);
 XmlText requestTypeValue = soapMessage.CreateTextNode("http://schemas.xmlsoap.org/ws/2005/02/trust/Issue");
 requestType.AppendChild(requestTypeValue);
 soapEnvelope.AppendChild(requestType);
 XmlElement keyType = soapMessage.CreateElement("t", "KeyType", soapMessage.DocumentElement.NamespaceURI);
 XmlText keyTypeValue = soapMessage.CreateTextNode("http://schemas.xmlsoap.org/ws/2005/05/identity/NoProofKey");
 keyType.AppendChild(keyTypeValue);
 soapEnvelope.AppendChild(keyType);

 return soapMessage.OuterXml;
 }

 private static bool ValidateRemoteCertificate(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors policyErrors)
 {
 // It may be insightful to examine the parameters here, while debugging, although the function only returns a value of true regardless of those parameters.
 return true;
 }
 }
}

Error: InfoPath Forms Services – Runtime – Data Connections – Remote server returned an error: (401) Unauthorized

I received the following error when my full-trust administrator-approved InfoPath form tried to call a web service data source, even in a single server farm:

InfoPath Forms Services                   Runtime – Data Connections            xxxx        Warning                …age: The remote server returned an error: (401) Unauthorized. The remote server returned an error: (401) Unauthorized.)

I discovered that there were two reasons for this error.  Either:

1. Claims authentication was enabled on the web application and should not have been. Disabling claims auth solved this issue. Had to use classic mode authentication. -or-
2. Kerberos needed to be enabled on the web application (for farms that have a load balancer involved).

Some red herrings (incorrect conclusions) included:

• Thinking that NT AUTHORITY\IUSR_xxxx needed to have permission somehow to the web service.  Not in my case.
• Thinking that the app pool account or any other accounts of InfoPath Forms Services or other needed to be explicitly added to the Members or Visitors SharePoint group in the site.  Not so in my case.

Batch-Populate InfoPath Forms in SharePoint from Flat File

The following code was a way that I devised to batch-load an InfoPath form library from a flat file, autopopulating both the InfoPath form data and the metadata of the SharePoint library.

Overview

The purpose of this code was to batch load flat file records into a SharePoint InfoPath form library, creating one document for each row of the flat file.

This builds on various techniques and uses LINQ to load the flat file, XML serializer and text writer to create and load an XML document whose schema is that of the InfoPath form and does an HTTP PUT to save the document to the form library in SharePoint for each row of the flat file.

Note that the display of the code below is cut off due to the WordPress template formatting.  Copy and paste the code to Visual Studio to read it more easily.

Directions

To experiment with this code, follow these steps:

1. Cut and paste the following code into a new Visual Studio console application.
2. Follow the directions in the code comments (in bold + italics + underline below) to create and add the myschema.XSD-as-C# class file to the console project.
3. Use the Ctrl+K+D to fix the code formatting in Visual Studio.  The indentation was lost on pasting this code into this blog entry.

Source code

 using System;
 using System.IO;
 using System.Linq;
 using System.Net;
 using System.Text;
 using System.Web;
 using System.Xml;
 using System.Xml.Linq;
 using System.Xml.Serialization;

 /*
 * NOTES ABOUT THIS CONSOLE APPLICATION:
 *
 * Put this project under C:\PROJECTS on your dev machine, if you like, to become
 * C:\PROJECTS\Load_InfoPath_and_Metadata_from_Flatfile.
 *
 * This project got a bit of code from the section 'Populate and Upload Form Data' in this
 * article: http://www.codeproject.com/KB/sharepoint/InfopathForm.aspx that I wish to
 * acknowledge. Also, since this app will probably not be run on a web front end but
 * rather the client, this app does not use SharePoint framework objects. it uses the PUT
 * technique found on the internet instead.
 *
 * This code assumes that you have an InfoPath form with five text fields (col1 - col5).
 * This code assumes that your InfoPath form template is already uploaded to the form
 * library and that you can create a new document by clicking New from the form library
 * menu.
 *
 * This project is a Windows console application. The schema file of the InfoPath form
 * was used to create a C-sharp code file using the command xsd.exe /c /l:CS myschema.xsd,
 * the latter of which can be obtained either by exporting the InfoPath form's source
 * files using the InfoPath Designer or by viewing the form library in Windows Explorer
 * view and examining the hidden Forms folder.
 *
 * If the InfoPath form template itself changes, you must save the source files of the
 * InfoPath form to a folder on disk and run the command xsd.exe /c /l:CS myschema.xsd
 * on the myschema.xsd file. This will create a myschema.cs C-sharp code file. Add the
 * updated file to this project. Recompile this project and fix anything that the 
 * schema change may have broken (if anything). When the schema changes, very likely
 * you will have to add new fields to the code of the "fields" object below so that the
 * data will propagate into SharePoint.
 */

 namespace Load_InfoPath_and_Metadata_from_Flatfile
 {
 class Program
 {
 static void Main(string[] args)
 {
 // Read flat file in its entirety.
 // TO-DO: CHANGE THE PATH AND FILENAME TO MATCH YOUR FILE.

 // Note that our sample flat file contains several lines with contents like this:
 // some data col 1|some data col 2|some data col 3|some data col 4|some data col 5
 // some data col 1|some data col 2|some data col 3|some data col 4|some data col 5
 // some data col 1|some data col 2|some data col 3|some data col 4|some data col 5

 string[] flatFileLines =
 File.ReadAllLines(@"C:\FlatfileDropFolder\SAMPLE_FLAT_FILE.txt");
 // Load the flat file recs into a variable that we will use below
 // (an enumerable collection of anonymous types) (LINQ stuff)
 var flatFileRecs = from line in flatFileLines
 let items = line.Split(new char[] { '|' })
 select new
 {
 col1 = items[0],
 col2 = items[1],
 col3 = items[2],
 col4 = items[3],
 col5 = items[4]
 };

 // Iterate through each of the records, building an XML (InfoPath) document and uploading
 // it to our Projects library

 foreach (var rec in flatFileRecs)
 {
 // Instantiate a myFields object in order to populate the data for this flat file rec,
 // then populate all its fields from the "rec" variable that contains a record from the
 // flat file that we just loaded.
 myFields fields = new myFields();
 fields.col1 = rec.col1;
 fields.col2 = rec.col2;
 fields.col3 = rec.col3;
 fields.col4 = rec.col4;
 fields.col5 = rec.col5;
 // Create a memory stream object
 MemoryStream mstream = new MemoryStream();
 // Initialize the save location
 string saveLoc = "http://xyz.abcdef.com/sites/somesitecoll/somesite/MyLibrary";
 using (mstream)
 {
 // Create serializer and writer objects
 XmlSerializer serializer = new XmlSerializer(typeof(myFields));
 XmlTextWriter writer = new XmlTextWriter(mstream, Encoding.UTF8);
 /*
 * Fill in the following portion of the document:
 *
 * <?mso-infoPathSolution
 * name="urn:schemas-microsoft-com:office:infopath:Projects:-myXSD-2011-01-27T05-51-32"
 * solutionVersion="1.0.0.25"
 * productVersion="12.0.0.0"
 * PIVersion="1.0.0.0"
 * href="http://xyz.abcdef.com/sites/somesitecoll/somesite/MyLibrary/Forms/template.xsn"?>
 * <?mso-application progid="InfoPath.Document" versionProgid="InfoPath.Document.2"?>
 * <my:myFields
 * xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 * xmlns:my="http://schemas.microsoft.com/office/infopath/2003/myXSD/2011-01-07T05:51:32"
 * xmlns:xd="http://schemas.microsoft.com/office/infopath/2003"
 * xml:lang="en-US">
 *
 */
 // Configure XML document legibility
 writer.Formatting = Formatting.Indented;
 writer.Indentation = 4;
 // Write the XML junk that always appears at the top of the document
 writer.WriteStartDocument();
 // Insert infopath processing instructions
 // (Get the correct parameters for this by examining a saved form in your library.)
 string msoInfoPathSolution =
 "name=\"urn:schemas-microsoft-com:office:infopath:Projects:-myXSD-2011-01-27T05-51-32\" " +
 "solutionVersion=\"1.0.0.25\" " +
 "productVersion=\"12.0.0.0\" " +
 "PIVersion=\"1.0.0.0\" " +
 "href=\"" + saveLoc + "/Forms/template.xsn\"";
 writer.WriteProcessingInstruction("mso-infoPathSolution", msoInfoPathSolution);
 writer.WriteProcessingInstruction("mso-application",
 "progid=\"InfoPath.Document\" versionProgid=\"InfoPath.Document.2\"");
 // Add namespaces
 XmlSerializerNamespaces ns = new XmlSerializerNamespaces();
 ns.Add("xsi", "http://www.w3.org/2001/XMLSchema-instance");
 ns.Add("my", "http://schemas.microsoft.com/office/infopath/2003/myXSD/2011-01-07T05:51:32");
 ns.Add("xd", "http://schemas.microsoft.com/office/infopath/2003");
 // Serialize the field data into the XML document, producing the data in XML elements
 // the way that InfoPath expects them
 serializer.Serialize(writer, fields, ns);
 // Close the document
 writer.WriteEndDocument();
 // Read the bytes out of the writer and into a byte array appropriately sized
 int len = (int)writer.BaseStream.Length;
 byte[] fileBytes = new byte[len];
 writer.BaseStream.Position = 0;
 writer.BaseStream.Read(fileBytes, 0, len);
 // Establish security context
 NetworkCredential netCred =
 new NetworkCredential("username_here", "password_here", "DOMAIN_HERE");
 // Upload the XML InfoPath document to the Projects library using HTTP PUT
 // (This means that access to a web front end is not needed.)
 WebClient webClient = new WebClient();
 webClient.Credentials = netCred;
 string saveFilename =
 HttpUtility.UrlPathEncode(saveLoc + "/" + fields.Project_Name + ".xml");
 byte[] udResult =
 webClient.UploadData(saveFilename, "PUT", fileBytes);
 }
 }
 }
 }
 }