Files
rockfish/util/FBImporter.cs
2018-06-28 23:37:38 +00:00

592 lines
26 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System;
using System.Text;
using System.Threading.Tasks;
using System.Collections.Generic;
using System.Linq;
using Microsoft.EntityFrameworkCore;
using rockfishCore.Models;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
//using System.Xml;
using System.Xml.Linq;
using System.IO;
namespace rockfishCore.Util
{
//Import fogbugz stuff into our own db
//called by schema update 7
//does not modify FB at all
//can be called any time, won't re-import
public static class FBImporter
{
public const long LAST_EXISTING_CASE_IN_FOGBUGZ = 3315;
public static void Import(rockfishContext ct)
{
//fogbugz api token, can get it by using the FB UI, open dev tools, be logged in, do something and watch the token get sent in the network traffic
string sToken = "jvmnpkrvc1i7hc6kn6dggqkibktgcc";
long nCase = 0;
bool bDone = false;
while (!bDone)
{
//increment until we reach the end (case 3309 as of now)
nCase++;
if (nCase > LAST_EXISTING_CASE_IN_FOGBUGZ)
{
bDone = true;
break;
}
//no need for this, we wipe clean before every import now
// //do we already have this record?
// if (ct.RfCase.Any(e => e.Id == nCase))
// {
// continue;
// }
fbCase fb = getCase(nCase, sToken);
//copy over to db
portCase(fb, ct);
}
}
private static Dictionary<string, long> dictProjects = new Dictionary<string, long>();
///////////////////////////////////////////////
//Port a single case over to Rockfish
//
private static void portCase(fbCase f, rockfishContext ct)
{
long lProjectId = 0;
if (f.noRecord)
{
//save as *DELETED* record
//Only setting required fields
RfCase c = new RfCase();
c.Priority = 5;
c.Title = "deleted from FogBugz";
c.RfCaseProjectId = 1;//has to be something
ct.RfCase.Add(c);
ct.SaveChanges();
}
else
{
if (dictProjects.ContainsKey(f.project))
{
lProjectId = dictProjects[f.project];
}
else
{
//need to insert it
RfCaseProject t = new RfCaseProject();
t.Name = f.project;
ct.RfCaseProject.Add(t);
ct.SaveChanges();
dictProjects.Add(f.project, t.Id);
lProjectId = t.Id;
}
RfCase c = new RfCase();
c.DtClosed = f.closed;
c.DtCreated = f.created;
c.Notes = f.notes;
c.Priority = f.priority;
if (c.Priority > 5)
c.Priority = 5;
c.Title = f.title;
c.RfCaseProjectId = lProjectId;
ct.RfCase.Add(c);
ct.SaveChanges();
if (f.attachments.Count > 0)
{
foreach (fbAttachment att in f.attachments)
{
RfCaseBlob blob = new RfCaseBlob();
blob.File = att.blob;
blob.Name = att.fileName;
blob.RfCaseId = c.Id;
ct.RfCaseBlob.Add(blob);
ct.SaveChanges();
}
}
}
// Console.WriteLine("***************************************************************************");
Console.WriteLine("~~~~~~~~~~~~~~~~~~~~ PORTCASE: " + f.id);
}
///////////////////////////////////////////////
//fetch a single case
//
private static fbCase getCase(long caseNumber, string sToken)
{
//Use this url
//https://fog.ayanova.com/api.asp?cmd=search&cols=ixBug,ixBugParent,sTitle,sProject,ixPriority, sPriority,fOpen,ixStatus,sStatus,dtOpened,dtResolved,dtClosed,ixRelatedBugs,events&token=jvmnpkrvc1i7hc6kn6dggqkibktgcc&q=3308
//TEST TEST TEST
// caseNumber = 3279;
string url = "https://fog.ayanova.com/api.asp?cmd=search&cols=ixBug,ixBugParent,sTitle,sProject,ixPriority,sPriority,fOpen,ixStatus,sStatus,dtOpened,dtResolved,dtClosed,ixRelatedBugs,events&token=" + sToken + "&q=" + caseNumber.ToString();
var httpClient = new HttpClient();
var result = httpClient.GetAsync(url).Result;
var stream = result.Content.ReadAsStreamAsync().Result;
var x = XElement.Load(stream);
fbCase f = new fbCase();
//are we done?
string scount = (from el in x.DescendantsAndSelf("cases") select el).First().Attribute("count").Value;
if (scount == "0")
{
f.noRecord = true;
return f;
}
//Got record, process it...
f.title = getValue("sTitle", x);
f.project = getValue("sProject", x);
f.id = getValue("ixBug", x);
f.priority = int.Parse(getValue("ixPriority", x));
f.created = Util.DateUtil.ISO8601StringToEpoch(getValue("dtOpened", x));
f.closed = Util.DateUtil.ISO8601StringToEpoch(getValue("dtClosed", x));
//string DTB4 = getValue("dtOpened", x);
//string DTAFTER = Util.DateUtil.ISO8601StringToLocalDateTime(DTB4);
//NOTES / ATTACHMENTS
StringBuilder sbNotes = new StringBuilder();
//events
var events = (from el in x.Descendants("event") select el).ToList();
bool bFirstNoteAdded = false;
foreach (var e in events)
{
string sNote = getValue("s", e);
if (!string.IsNullOrWhiteSpace(sNote))
{
if (bFirstNoteAdded)
{
sbNotes.AppendLine("====================");
}
bFirstNoteAdded = true;
sbNotes.Append(Util.DateUtil.ISO8601StringToLocalDateTime(getValue("dt", e)));
sbNotes.Append(" ");
sbNotes.AppendLine(getValue("evtDescription", e));
sbNotes.AppendLine("____________________");
sbNotes.AppendLine(sNote);
}
//GET ATTACHMENTS
var attaches = (from l in e.Descendants("attachment") select l).ToList();
foreach (var a in attaches)
{
fbAttachment fbat = new fbAttachment();
fbat.fileName = getValue("sFileName", a);
string fileUrl = "https://fog.ayanova.com/" + System.Net.WebUtility.HtmlDecode(getValue("sURL", a)) + "&token=" + sToken;
/////
using (var aclient = new HttpClient())
{
var aresponse = aclient.GetAsync(fileUrl).Result;
if (aresponse.IsSuccessStatusCode)
{
// by calling .Result you are performing a synchronous call
var responseContent = aresponse.Content;
// by calling .Result you are synchronously reading the result
var astream = responseContent.ReadAsStreamAsync().Result;
MemoryStream ams = new MemoryStream();
astream.CopyTo(ams);
fbat.blob = ams.ToArray();
}
}
/////
//bugbug: Looks like this is always the same size, maybe it's getting an error page?
// result = httpClient.GetAsync(url).Result;
// stream = result.Content.ReadAsStreamAsync().Result;
// MemoryStream ms = new MemoryStream();
// stream.CopyTo(ms);
// fbat.blob = ms.ToArray();
f.attachments.Add(fbat);
}
}//bottom of events loop
if (sbNotes.Length > 0)
{
f.notes = sbNotes.ToString();
}
return f;
}
/////////////////////////////////
// Get header element value
//
private static string getValue(string sItem, XElement x)
{
return (string)(from el in x.Descendants(sItem) select el).First();
}
/////////////////////////////////////
//dto object
public class fbCase
{
public fbCase()
{
attachments = new List<fbAttachment>();
}
public bool noRecord;
public string id;
public string title;
public string project;
public int priority;
public string notes;
public long? created;
public long? closed;
public List<fbAttachment> attachments;
}
public class fbAttachment
{
public string fileName;
public byte[] blob;
}
//eoc
}
//eons
}
/*
<?xml version="1.0" encoding="UTF-8"?>
<response>
<cases count="1">
<case ixBug="3308" operations="edit,assign,resolve,remind">
<ixBug>3308</ixBug>
<ixBugParent>0</ixBugParent>
<sTitle><![CDATA[Port all Fogbugz functionality we use to Rockfish and decommission FogBugz]]></sTitle>
<sProject><![CDATA[RockFish]]></sProject>
<ixPriority>1</ixPriority>
<fOpen>true</fOpen>
<ixStatus>20</ixStatus>
<sStatus><![CDATA[Active]]></sStatus>
<dtOpened>2017-08-06T19:11:54Z</dtOpened>
<dtResolved></dtResolved>
<dtClosed></dtClosed>
<ixRelatedBugs>3240</ixRelatedBugs>
<events>
<event ixBugEvent="18150" ixBug="3308">
<ixBugEvent>18150</ixBugEvent>
<evt>1</evt>
<sVerb><![CDATA[Opened]]></sVerb>
<ixPerson>3</ixPerson>
<ixPersonAssignedTo>0</ixPersonAssignedTo>
<dt>2017-08-06T19:11:54Z</dt>
<s><![CDATA[http://help.fogcreek.com/the-fogbugz-api
And porting all existing data to rf including old closed cases etc.
Also must be searchable, generate case numbers, have category / project, and priority, accept file attachments like screenshots etc.
]]></s>
<fEmail>false</fEmail>
<fHTML>false</fHTML>
<fExternal>false</fExternal>
<sChanges></sChanges>
<sFormat></sFormat>
<rgAttachments></rgAttachments>
<evtDescription><![CDATA[Opened by John Cardinal]]></evtDescription>
<bEmail>false</bEmail>
<bExternal>false</bExternal>
<sPerson><![CDATA[John Cardinal]]></sPerson>
<sHtml><![CDATA[http://help.fogcreek.com/the-fogbugz-api<br />And porting all existing data to rf including old closed cases etc.<br />Also must be searchable, generate case numbers, have category / project, and priority, accept file attachments like screenshots etc.<br />]]></sHtml>
</event>
<event ixBugEvent="18151" ixBug="3308">
<ixBugEvent>18151</ixBugEvent>
<evt>3</evt>
<sVerb><![CDATA[Assigned]]></sVerb>
<ixPerson>3</ixPerson>
<ixPersonAssignedTo>3</ixPersonAssignedTo>
<dt>2017-08-06T19:11:55Z</dt>
<s></s>
<fEmail>false</fEmail>
<fHTML>false</fHTML>
<fExternal>false</fExternal>
<sChanges></sChanges>
<sFormat></sFormat>
<rgAttachments></rgAttachments>
<evtDescription><![CDATA[Assigned to John Cardinal by John Cardinal]]></evtDescription>
<bEmail>false</bEmail>
<bExternal>false</bExternal>
<sPerson><![CDATA[John Cardinal]]></sPerson>
<sHtml></sHtml>
</event>
<event ixBugEvent="18152" ixBug="3308">
<ixBugEvent>18152</ixBugEvent>
<evt>2</evt>
<sVerb><![CDATA[Edited]]></sVerb>
<ixPerson>3</ixPerson>
<ixPersonAssignedTo>0</ixPersonAssignedTo>
<dt>2017-08-07T21:18:55Z</dt>
<s><![CDATA[This query gets all the required data:
https://fog.ayanova.com/api.asp?cmd=search&cols=ixBug,sTitle,fOpen,ixStatus,dtOpened,dtResolved,dtClosed,ixRelatedBugs,events&token=jvmnpkrvc1i7hc6kn6dggqkibktgcc&q=3240
This is for case 3240 (q=3240) and the token I got from just viewing the network traffic using Fogbugz (which works fine).
The response is an xml document with a case/cases/case header and then an events collection under that which contains all the text added and other events like chaging the title etc which we don't care about. Also it contains urls for attachments to download and can use the url directly to get the attachment.
]]></s>
<fEmail>false</fEmail>
<fHTML>false</fHTML>
<fExternal>false</fExternal>
<sChanges></sChanges>
<sFormat></sFormat>
<rgAttachments></rgAttachments>
<evtDescription><![CDATA[Edited by John Cardinal]]></evtDescription>
<bEmail>false</bEmail>
<bExternal>false</bExternal>
<sPerson><![CDATA[John Cardinal]]></sPerson>
<sHtml><![CDATA[This query gets all the required data:<br />https://fog.ayanova.com/api.asp?cmd=search&amp;cols=ixBug,sTitle,fOpen,ixStatus,dtOpened,dtResolved,dtClosed,ixRelatedBugs,events&amp;token=jvmnpkrvc1i7hc6kn6dggqkibktgcc&amp;q=3240<br /><br />This is for case 3240 (q=3240) and the token I got from just viewing the network traffic using Fogbugz (which works fine).<br /><br /><br />The response is an xml document with a case/cases/case header and then an events collection under that which contains all the text added and other events like chaging the title etc which we don't care about.&nbsp; Also it contains urls for attachments to download and can use the url directly to get the attachment.<br /><br />]]></sHtml>
</event>
<event ixBugEvent="18153" ixBug="3308">
<ixBugEvent>18153</ixBugEvent>
<evt>2</evt>
<sVerb><![CDATA[Edited]]></sVerb>
<ixPerson>3</ixPerson>
<ixPersonAssignedTo>0</ixPersonAssignedTo>
<dt>2017-08-07T21:21:02Z</dt>
<s><![CDATA[As of right now this is the highest case number in use.
The procedure will be to start at 1 and fetch every case +1 until we get a response like this:
<?xml version="1.0" encoding="UTF-8"?><response><cases count="0"></cases></response>
Where cases count=0 and would normally be one and the cases branch would have the one case fetched.]]></s>
<fEmail>false</fEmail>
<fHTML>false</fHTML>
<fExternal>false</fExternal>
<sChanges></sChanges>
<sFormat></sFormat>
<rgAttachments></rgAttachments>
<evtDescription><![CDATA[Edited by John Cardinal]]></evtDescription>
<bEmail>false</bEmail>
<bExternal>false</bExternal>
<sPerson><![CDATA[John Cardinal]]></sPerson>
<sHtml><![CDATA[As of right now this is the highest case number in use.<br /><br />The procedure will be to start at 1 and fetch every case +1 until we get a response like this:<br />&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&lt;response&gt;&lt;cases count=&quot;0&quot;&gt;&lt;/cases&gt;&lt;/response&gt;<br /><br />Where cases count=0 and would normally be one and the cases branch would have the one case fetched.]]></sHtml>
</event>
<event ixBugEvent="18154" ixBug="3308">
<ixBugEvent>18154</ixBugEvent>
<evt>2</evt>
<sVerb><![CDATA[Edited]]></sVerb>
<ixPerson>3</ixPerson>
<ixPersonAssignedTo>0</ixPersonAssignedTo>
<dt>2017-08-07T21:24:48Z</dt>
<s><![CDATA[Fields to keep from old fogbugz:
PROJECTS - all of these in a table
CASES - every case:
CASE HEADER:
Case # (id), Title, project, priority
CASE EVENTS:
- s (string of text (CDATA))
- Any attachments
]]></s>
<fEmail>false</fEmail>
<fHTML>false</fHTML>
<fExternal>false</fExternal>
<sChanges></sChanges>
<sFormat></sFormat>
<rgAttachments></rgAttachments>
<evtDescription><![CDATA[Edited by John Cardinal]]></evtDescription>
<bEmail>false</bEmail>
<bExternal>false</bExternal>
<sPerson><![CDATA[John Cardinal]]></sPerson>
<sHtml><![CDATA[Fields to keep from old fogbugz:<br /><br />PROJECTS - all of these in a table<br /><br />CASES - every case:<br /><br />CASE HEADER:<br />Case # (id), Title, project, priority<br />&nbsp; CASE EVENTS:<br />&nbsp; &nbsp; - s (string of text (CDATA))<br />&nbsp; &nbsp; - Any attachments<br /><br />]]></sHtml>
</event>
<event ixBugEvent="18155" ixBug="3308">
<ixBugEvent>18155</ixBugEvent>
<evt>2</evt>
<sVerb><![CDATA[Edited]]></sVerb>
<ixPerson>3</ixPerson>
<ixPersonAssignedTo>0</ixPersonAssignedTo>
<dt>2017-08-07T21:34:27Z</dt>
<s></s>
<fEmail>false</fEmail>
<fHTML>false</fHTML>
<fExternal>false</fExternal>
<sChanges><![CDATA[Priority changed from '3 3' to '1 1'.
]]></sChanges>
<sFormat></sFormat>
<rgAttachments></rgAttachments>
<evtDescription><![CDATA[Edited by John Cardinal]]></evtDescription>
<bEmail>false</bEmail>
<bExternal>false</bExternal>
<sPerson><![CDATA[John Cardinal]]></sPerson>
<sHtml></sHtml>
</event>
<event ixBugEvent="18156" ixBug="3308">
<ixBugEvent>18156</ixBugEvent>
<evt>2</evt>
<sVerb><![CDATA[Edited]]></sVerb>
<ixPerson>3</ixPerson>
<ixPersonAssignedTo>0</ixPersonAssignedTo>
<dt>2017-08-07T21:37:37Z</dt>
<s><![CDATA[New will be primarily a single table, all events in fb now a single text field with date time headered sections for the events and hr lines or something.
New db tables:
CASE
CASEPROJECT
CASEBLOB
CASE Fields:
=-=-=-=-=-=-
ID (integer not null, case # autoincrement),
TITLE text not null,
PROJECT (fk CASEPROJECT not null),
PRIORITY (integer 1-5 not null),
NOTES (text nullable),
CREATED (date time as integer, nullable),
CLOSED (date time as integer, nullable, also serves as closed indicator or open indicator if null),
RELEASEVERSION (text nullable, public release version that fixed the item, i.e. if it was qboi 7.5 patch 1 then this would be 7.5.1),
RELEASENOTES (text nullable, single string of text that serves as customer description of fix).
CASEPROJECT fields:
=-=-=-=-=-=-=-=-=-=-
id (integer not null pk autoincrement),
name (text not null, project name i.e. QBOI)
CASEBLOB fields
id (integer not null pk autoincrement)
CASEID (fk not null link to case)
FILE (blob not null)
]]></s>
<fEmail>false</fEmail>
<fHTML>false</fHTML>
<fExternal>false</fExternal>
<sChanges></sChanges>
<sFormat></sFormat>
<rgAttachments></rgAttachments>
<evtDescription><![CDATA[Edited by John Cardinal]]></evtDescription>
<bEmail>false</bEmail>
<bExternal>false</bExternal>
<sPerson><![CDATA[John Cardinal]]></sPerson>
<sHtml><![CDATA[New will be primarily a single table, all events in fb now a single text field with date time headered sections for the events and hr lines or something. <br /><br />New db tables:<br />CASE<br />CASEPROJECT<br />CASEBLOB<br /><br />CASE Fields:<br />=-=-=-=-=-=-<br />ID (integer not null, case # autoincrement), <br />TITLE text not null, <br />PROJECT (fk CASEPROJECT not null), <br />PRIORITY (integer 1-5 not null), <br />NOTES (text nullable), <br />CREATED (date time as integer, nullable), <br />CLOSED (date time as integer, nullable, also serves as closed indicator or open indicator if null), <br />RELEASEVERSION (text nullable, public release version that fixed the item, i.e. if it was qboi 7.5 patch 1 then this would be 7.5.1),<br />RELEASENOTES (text nullable, single string of text that serves as customer description of fix).<br /><br />CASEPROJECT fields:<br />=-=-=-=-=-=-=-=-=-=-<br />id (integer not null pk autoincrement),<br />name (text not null, project name i.e. QBOI)<br /><br />CASEBLOB fields<br />id (integer not null pk autoincrement)<br />CASEID (fk not null link to case)<br />FILE (blob not null)<br /><br />]]></sHtml>
</event>
<event ixBugEvent="18157" ixBug="3308">
<ixBugEvent>18157</ixBugEvent>
<evt>2</evt>
<sVerb><![CDATA[Edited]]></sVerb>
<ixPerson>3</ixPerson>
<ixPersonAssignedTo>0</ixPersonAssignedTo>
<dt>2017-08-07T22:13:24Z</dt>
<s></s>
<fEmail>false</fEmail>
<fHTML>false</fHTML>
<fExternal>false</fExternal>
<sChanges><![CDATA[Revised John Cardinal's entry from 8/7/2017 at 9:37 PM UTC
]]></sChanges>
<sFormat></sFormat>
<rgAttachments></rgAttachments>
<evtDescription><![CDATA[Edited by John Cardinal]]></evtDescription>
<bEmail>false</bEmail>
<bExternal>false</bExternal>
<sPerson><![CDATA[John Cardinal]]></sPerson>
<sHtml></sHtml>
</event>
<event ixBugEvent="18158" ixBug="3308">
<ixBugEvent>18158</ixBugEvent>
<evt>2</evt>
<sVerb><![CDATA[Edited]]></sVerb>
<ixPerson>3</ixPerson>
<ixPersonAssignedTo>0</ixPersonAssignedTo>
<dt>2017-08-07T22:13:32Z</dt>
<s><![CDATA[UI will have option to "Append" and it will auto insert the date/time HR and then the text.]]></s>
<fEmail>false</fEmail>
<fHTML>false</fHTML>
<fExternal>false</fExternal>
<sChanges></sChanges>
<sFormat></sFormat>
<rgAttachments></rgAttachments>
<evtDescription><![CDATA[Edited by John Cardinal]]></evtDescription>
<bEmail>false</bEmail>
<bExternal>false</bExternal>
<sPerson><![CDATA[John Cardinal]]></sPerson>
<sHtml><![CDATA[UI will have option to &quot;Append&quot; and it will auto insert the date/time HR and then the text.]]></sHtml>
</event>
<event ixBugEvent="18159" ixBug="3308">
<ixBugEvent>18159</ixBugEvent>
<evt>2</evt>
<sVerb><![CDATA[Edited]]></sVerb>
<ixPerson>3</ixPerson>
<ixPersonAssignedTo>0</ixPersonAssignedTo>
<dt>2017-08-07T22:29:29Z</dt>
<s></s>
<fEmail>false</fEmail>
<fHTML>false</fHTML>
<fExternal>false</fExternal>
<sChanges><![CDATA[Revised John Cardinal's entry from 8/7/2017 at 9:37 PM UTC
]]></sChanges>
<sFormat></sFormat>
<rgAttachments></rgAttachments>
<evtDescription><![CDATA[Edited by John Cardinal]]></evtDescription>
<bEmail>false</bEmail>
<bExternal>false</bExternal>
<sPerson><![CDATA[John Cardinal]]></sPerson>
<sHtml></sHtml>
</event>
<event ixBugEvent="18160" ixBug="3308">
<ixBugEvent>18160</ixBugEvent>
<evt>2</evt>
<sVerb><![CDATA[Edited]]></sVerb>
<ixPerson>3</ixPerson>
<ixPersonAssignedTo>0</ixPersonAssignedTo>
<dt>2017-08-07T22:49:22Z</dt>
<s></s>
<fEmail>false</fEmail>
<fHTML>false</fHTML>
<fExternal>false</fExternal>
<sChanges><![CDATA[Title changed from 'Look into porting Fogbugz functionality we use to Rockfish' to 'Port all Fogbugz functionality we use to Rockfish and decommission FogBugz'.
]]></sChanges>
<sFormat></sFormat>
<rgAttachments></rgAttachments>
<evtDescription><![CDATA[Edited by John Cardinal]]></evtDescription>
<bEmail>false</bEmail>
<bExternal>false</bExternal>
<sPerson><![CDATA[John Cardinal]]></sPerson>
<sHtml></sHtml>
</event>
</events>
</case>
</cases>
</response>
*/