1

I have the following list and I am trying to create something similar to the following (Not the ROC curve, just the standard deviation part);

enter image description here

The image comes from the following link (written in Python) link

Structure:

List of 2
 $ newcol    :List of 6
  ..$ : num [1:75] 0.887 0.903 0.909 0.912 0.914 ...
  ..$ : num [1:75] 0.921 0.939 0.947 0.951 0.954 ...
  ..$ : num [1:75] 0.887 0.906 0.912 0.915 0.917 ...
  ..$ : num [1:75] 0.917 0.939 0.947 0.951 0.954 ...
  ..$ : num [1:75] 0.889 0.907 0.913 0.917 0.92 ...
  ..$ : num [1:75] 0.921 0.942 0.95 0.954 0.957 ...
 $ newcol_std:List of 6
  ..$ : num [1:75] 0.00409 0.00273 0.00269 0.00343 0.00229 ...
  ..$ : num [1:75] 0.002934 0.002618 0.001522 0.001453 0.000926 ...
  ..$ : num [1:75] 0.00357 0.00153 0.00151 0.00134 0.0012 ...
  ..$ : num [1:75] 0.003542 0.002235 0.000998 0.000965 0.000733 ...
  ..$ : num [1:75] 0.0037 0.002689 0.001815 0.001497 0.000975 ...
  ..$ : num [1:75] 0.0024 0.0015 0.00145 0.00147 0.00135 ...

Data:

structure(list(newcol = list(c(0.8872707, 0.9030507, 0.9087002, 
0.9121047, 0.9138395, 0.9152718, 0.9158744, 0.9171358, 0.9177393, 
0.9183246, 0.9187044, 0.9189548, 0.9193481, 0.9197915, 0.9200788, 
0.9204141, 0.9205686, 0.9207148, 0.9211112, 0.9212452, 0.9214546, 
0.9216964, 0.9218029, 0.9219894, 0.9221818, 0.9223518, 0.9224333, 
0.9227003, 0.9228383, 0.923039, 0.9231552, 0.9233157, 0.9235961, 
0.9238084, 0.9239563, 0.9241449, 0.9242382, 0.9243814, 0.9245184, 
0.9246821, 0.9248516, 0.9249441, 0.9250369, 0.9251405, 0.9253599, 
0.9255014, 0.9256391, 0.9257434, 0.9258243, 0.925896, 0.9259839, 
0.9261379, 0.9262587, 0.9263849, 0.9265621, 0.9266335, 0.9268006, 
0.9268646, 0.9269769, 0.9271026, 0.927221, 0.9273457, 0.9274789, 
0.9276086, 0.9276941, 0.9278218, 0.9279294, 0.928021, 0.9281182, 
0.9282149, 0.9283187, 0.928444, 0.9285454, 0.9286198, 0.9287347
), c(0.9206486, 0.9394105, 0.9466493, 0.9507703, 0.9537551, 0.9552856, 
0.9564934, 0.9570751, 0.9575526, 0.9581214, 0.9586022, 0.9590438, 
0.9593632, 0.9597253, 0.9600932, 0.9603035, 0.9606403, 0.9608917, 
0.961082, 0.9613016, 0.9615447, 0.9617322, 0.9618319, 0.9620238, 
0.9622155, 0.9623541, 0.9624744, 0.9625615, 0.9627035, 0.9628195, 
0.9629159, 0.9630224, 0.9631543, 0.9633178, 0.9634451, 0.9635651, 
0.9637093, 0.9638339, 0.963972, 0.9640747, 0.9642088, 0.9643436, 
0.9644267, 0.9645442, 0.9646703, 0.9647597, 0.9648466, 0.9649245, 
0.9650489, 0.9651353, 0.9652623, 0.9653706, 0.9654856, 0.9656194, 
0.9657087, 0.9658079, 0.9659056, 0.9660122, 0.9661219, 0.9661943, 
0.9662854, 0.9664101, 0.9665026, 0.9666136, 0.9667218, 0.9668299, 
0.9669346, 0.9670301, 0.9671197, 0.9672051, 0.9672939, 0.9674228, 
0.9675352, 0.9676268, 0.9677075), c(0.8867506, 0.9062565, 0.9124712, 
0.9149032, 0.9171106, 0.9188126, 0.9202478, 0.9214695, 0.9222806, 
0.9230098, 0.9236721, 0.9245489, 0.9253719, 0.9260546, 0.9266058, 
0.9271845, 0.9277118, 0.9282387, 0.9287738, 0.9293859, 0.9298391, 
0.9303543, 0.9309485, 0.931473, 0.9320202, 0.9326178, 0.9331355, 
0.9337554, 0.9342853, 0.9347901, 0.9353138, 0.9357456, 0.9362311, 
0.9366839, 0.9371574, 0.9375803, 0.9379687, 0.938406, 0.9388169, 
0.939192, 0.9395761, 0.9399526, 0.9403174, 0.9407229, 0.9410972, 
0.9414594, 0.9417982, 0.9421476, 0.9424574, 0.9427787, 0.943154, 
0.943464, 0.9437745, 0.9440851, 0.9443794, 0.9446718, 0.9449876, 
0.9452686, 0.9455363, 0.9458454, 0.9461365, 0.9464448, 0.946717, 
0.9469806, 0.9472517, 0.9475034, 0.9477519, 0.9480087, 0.9482629, 
0.9485097, 0.9487402, 0.94897, 0.949203, 0.9494276, 0.9496601
), c(0.9169705, 0.9391633, 0.9473486, 0.9514407, 0.9544649, 0.9563262, 
0.9580368, 0.9591332, 0.9603098, 0.9613646, 0.9622262, 0.9629543, 
0.9636388, 0.9642505, 0.9648797, 0.9654704, 0.9660472, 0.9666298, 
0.9671768, 0.9677175, 0.9681838, 0.9686972, 0.9691675, 0.969661, 
0.9701463, 0.9706446, 0.9710811, 0.9715778, 0.9720197, 0.9724278, 
0.9728217, 0.973216, 0.9736271, 0.9739941, 0.974367, 0.9747324, 
0.9750415, 0.9753976, 0.9757126, 0.9760339, 0.9763586, 0.9766528, 
0.9769567, 0.9772719, 0.9775654, 0.9778746, 0.9781418, 0.9784301, 
0.9787192, 0.9790325, 0.9793007, 0.9795395, 0.9797859, 0.9800408, 
0.9802992, 0.980527, 0.9807694, 0.9809618, 0.981154, 0.9813642, 
0.9815924, 0.9818347, 0.9820304, 0.9822305, 0.9824408, 0.9826247, 
0.9828244, 0.9830079, 0.9831818, 0.9833603, 0.9835534, 0.9837096, 
0.9838724, 0.984035, 0.9841532), c(0.8885806, 0.9068375, 0.912755, 
0.9165201, 0.919588, 0.9216641, 0.9234095, 0.9251735, 0.9263224, 
0.9280197, 0.9293092, 0.9305007, 0.9317485, 0.9327461, 0.9337898, 
0.9348142, 0.9357507, 0.936578, 0.9374376, 0.9382772, 0.9390516, 
0.939838, 0.9405919, 0.9412297, 0.9419274, 0.9425768, 0.9432503, 
0.9438852, 0.9444603, 0.9450414, 0.9455898, 0.9461341, 0.9466634, 
0.9471886, 0.9476879, 0.9481665, 0.9485791, 0.9490332, 0.9494726, 
0.9499595, 0.9503578, 0.9507834, 0.9511654, 0.9515419, 0.9518568, 
0.9521849, 0.952515, 0.9528262, 0.9532093, 0.953576, 0.9538625, 
0.9542301, 0.9545868, 0.9549169, 0.9552529, 0.9555878, 0.955866, 
0.9561445, 0.9564196, 0.9567375, 0.9570449, 0.9573439, 0.9576419, 
0.9579816, 0.9581898, 0.9584926, 0.9587513, 0.9589362, 0.959221, 
0.9594918, 0.959786, 0.9600281, 0.9603172, 0.9605213, 0.9607696
), c(0.9209257, 0.9416686, 0.94968, 0.953754, 0.9567425, 0.9592916, 
0.9610749, 0.9627777, 0.9642636, 0.9655278, 0.9667164, 0.9677239, 
0.9686608, 0.9696417, 0.9705726, 0.9715037, 0.972335, 0.9731109, 
0.9738291, 0.9745612, 0.9752563, 0.9759226, 0.9765843, 0.977179, 
0.9778345, 0.9783888, 0.978952, 0.9794643, 0.979904, 0.980296, 
0.980754, 0.9811558, 0.9815654, 0.9818968, 0.9822738, 0.9826753, 
0.9830283, 0.9833548, 0.9836681, 0.9839338, 0.9842503, 0.9844832, 
0.9847261, 0.9849646, 0.9852087, 0.9855209, 0.9857539, 0.9859616, 
0.9861817, 0.9864459, 0.9866604, 0.9868963, 0.9870737, 0.9872816, 
0.9874609, 0.9876679, 0.9878247, 0.988032, 0.9882744, 0.9884803, 
0.9886296, 0.9888351, 0.988999, 0.9892102, 0.9893545, 0.9895261, 
0.9896919, 0.9898998, 0.9900405, 0.9901832, 0.9903173, 0.9904538, 
0.9906236, 0.9908199, 0.9909622)), newcol_std = list(c(0.00409164817769325, 
0.00272534944732355, 0.00268643651702685, 0.00343004478250996, 
0.0022860659767556, 0.00241296861976596, 0.00222128094576296, 
0.00178995205517931, 0.00141983830415714, 0.00109001726589091, 
0.000947828170092983, 0.000887689450253728, 0.000824407478141823, 
0.000833376895583693, 0.00100469814372754, 0.000968352358364249, 
0.000898566769955244, 0.000906003730668586, 0.000780152651717826, 
0.000869937790836819, 0.000841395412439939, 0.000907521041128739, 
0.00101517855080476, 0.00109289461527849, 0.00120329554146209, 
0.00116827289624674, 0.00116662805126766, 0.00120320547290984, 
0.00115794240357024, 0.00112805779997068, 0.00109230570809949, 
0.00111398528265399, 0.00107557030920684, 0.00107203294731104, 
0.00110265053852582, 0.00108036646101073, 0.0010727639815101, 
0.0010486929198004, 0.00111752174032403, 0.00118130677222912, 
0.00115031936433829, 0.00114325722822673, 0.00114927389689859, 
0.0011413578974346, 0.00107869935105457, 0.00103131975640197, 
0.00101405911563126, 0.000997056186948478, 0.000964826518156463, 
0.00098133266527011, 0.000991103874500537, 0.000983863552558259, 
0.000993576675480126, 0.00100379364915502, 0.000914261718584059, 
0.000875307517364319, 0.000844845571743433, 0.000848408415812084, 
0.000850688127356881, 0.00082228136304473, 0.00081209137425208, 
0.0008457489047102, 0.000849723890457742, 0.000881503964796988, 
0.000908543058885886, 0.000897146008179109, 0.000893916349563863, 
0.000904692876047002, 0.000900858790240409, 0.000928360646526741, 
0.000938265532847713, 0.000937514586559892, 0.000944497347805172, 
0.000919230526065756, 0.000914587016067583), c(0.00293449110406852, 
0.00261826726097734, 0.00152184651328577, 0.00145304040198989, 
0.000926091512795068, 0.00101862330626016, 0.00099282064840809, 
0.000935019513214243, 0.000883342877942995, 0.00105515612113772, 
0.00108486799199471, 0.00109938417305741, 0.000983145238536673, 
0.000887015901802552, 0.000716852816108211, 0.000677968915235634, 
0.00070231418175788, 0.000660465449504358, 0.000671726879011075, 
0.00074326619729005, 0.000718163637303307, 0.000736582758373035, 
0.000756198578408127, 0.000763887661860564, 0.000751948967697321, 
0.000756538227711604, 0.000742497434361065, 0.000751140100117713, 
0.000756795249770621, 0.000741437826143756, 0.000703499459773145, 
0.000729130742688073, 0.000749910534684413, 0.000714615952777288, 
0.000713036387531649, 0.00073119264903909, 0.00073299441331672, 
0.000733034439855545, 0.000708088130053587, 0.000665471118749558, 
0.000668574274182821, 0.000705294151469428, 0.000666840468152049, 
0.000681527666385967, 0.000661112857252269, 0.000603036325585754, 
0.000609008078705972, 0.000621338917174145, 0.000617040914312639, 
0.000581556540614263, 0.000562806902989098, 0.000573605997233441, 
0.000565380791940642, 0.000570956075358564, 0.000578558562328083, 
0.000594920910650473, 0.000605676184036962, 0.000583567442630618, 
0.000576856212658665, 0.000566185314291247, 0.000574172134515219, 
0.000570728385325591, 0.000547321514204814, 0.000523006347980033, 
0.000510257150861249, 0.000510098902121436, 0.000486764255125707, 
0.000492069598794836, 0.000485354313778258, 0.000501198653181825, 
0.000503838356898448, 0.000498638907296731, 0.000512919837834276, 
0.000510557303258398, 0.000481195230516309), c(0.00357292844035119, 
0.00152820190092506, 0.00150669909402023, 0.00134425487163985, 
0.00120355101264022, 0.00115450978345798, 0.00108853946181312, 
0.00128970851365139, 0.00126234965044389, 0.0012223339805425, 
0.00118923340434798, 0.00101057354507749, 0.00102682339764107, 
0.000912698986452827, 0.000859693643046411, 0.000731094693007949, 
0.00063625384872731, 0.000643503698611197, 0.000698240187897095, 
0.000672332574039516, 0.000612109540848898, 0.000628358504226431, 
0.000607687460793624, 0.00060909473815273, 0.00054440422491667, 
0.000551438264870423, 0.000561954313116002, 0.000474506522544131, 
0.000421043952633831, 0.000448005457626932, 0.000389745506705483, 
0.000393382307815442, 0.000377283832123274, 0.000427893549929719, 
0.000423990848911491, 0.000453724376636996, 0.000455090111940782, 
0.000482230650234134, 0.0004595820818288, 0.000472018431839358, 
0.000440883306381774, 0.000460052649156707, 0.000443772960069364, 
0.000434926303919156, 0.000393523518983643, 0.000399067963991548, 
0.000421364165465708, 0.000470482773433298, 0.000439766119734913, 
0.00044021632188324, 0.000434337886764765, 0.000434200184367479, 
0.00040616480659189, 0.00042151380765637, 0.000392676508233522, 
0.000400459685889368, 0.000416588573861773, 0.000408103222208649, 
0.000448771445202199, 0.000458729375657268, 0.000470055794681863, 
0.00046143467583536, 0.000460783029205493, 0.000431383634552493, 
0.000415965154720969, 0.000417753084974685, 0.000400069606487379, 
0.000403557443161626, 0.00043617277535085, 0.000417131406111669, 
0.00041202276631348, 0.000419960474393929, 0.000426861101519506, 
0.000460899381642515, 0.000469826446668015), c(0.00354165188153849, 
0.0022349458181184, 0.000997580994217157, 0.000964700787796788, 
0.00073306567921624, 0.000710233313710729, 0.000625398081279726, 
0.000490628127957095, 0.000506804064794208, 0.000575933190580837, 
0.00042704772562339, 0.000430620261917415, 0.000538531856058212, 
0.00055094886329955, 0.000541521938641662, 0.000506945401324812, 
0.000436335146507066, 0.000452529954861714, 0.000468762370623022, 
0.000488813103305814, 0.000401088219614221, 0.000401301083918401, 
0.000398048301150656, 0.000445681949345642, 0.000448801526274917, 
0.00042674587286068, 0.000428669674636856, 0.000420402854557802, 
0.000401207440194931, 0.000344873832216061, 0.000323097833461446, 
0.000279252573989175, 0.000253421960482719, 0.000268446437980032, 
0.000256582150670301, 0.000247946042447571, 0.000248845835932648, 
0.000232168990082558, 0.000292588516504404, 0.000299311693694051, 
0.000294725363845882, 0.000286559173614947, 0.000251507475230065, 
0.000282349942582016, 0.000268195152909458, 0.000250460056478974, 
0.000253692254664789, 0.000271448503408793, 0.000279123198564345, 
0.000277203986364399, 0.000294534225458843, 0.00031651516557332, 
0.000306287920178622, 0.000308425290709291, 0.000311434038125927, 
0.000339324917991653, 0.000350336752191128, 0.000319548368925524, 
0.000362867469067881, 0.000363952139613985, 0.000352562675352853, 
0.00035783852486191, 0.000355654664049202, 0.000374074658318586, 
0.000369409203992363, 0.000377815311969847, 0.000369389279896289, 
0.000359742254973261, 0.000347287143552535, 0.00035681817507479, 
0.000342749821133584, 0.000378894233401069, 0.000380271271717154, 
0.000365651200049844, 0.000363856235479093), c(0.00370018740607045, 
0.00268850710433513, 0.00181547839429551, 0.00149692734957841, 
0.000975494131243777, 0.000650016222877862, 0.000773948867826388, 
0.000831897619934811, 0.000740837796043558, 0.000716199001674962, 
0.000854354001538811, 0.000698135237615327, 0.000540922036914763, 
0.00051339642567092, 0.00064497702279769, 0.000641402338671974, 
0.000618328399698538, 0.000632301510314925, 0.00057691302645439, 
0.000522780030134534, 0.000451692196203872, 0.000381833209535397, 
0.000431233451869555, 0.000461468102963264, 0.000390356554992864, 
0.000363284406541168, 0.000363014063072818, 0.000357338998681945, 
0.000380777113332974, 0.000407612119383663, 0.000436426809411314, 
0.000391613189382482, 0.000431122534748407, 0.000421787434639363, 
0.000474389597294252, 0.000459979836423244, 0.000456555023995053, 
0.000455206722230648, 0.000458215495046741, 0.00049023958430678, 
0.000539361622650343, 0.000554869209874598, 0.000492904493703385, 
0.00044823842977795, 0.00045133287050915, 0.000422196743262655, 
0.000465229620770756, 0.000452887138158714, 0.000532153558633451, 
0.000508376238684291, 0.000543769114549573, 0.000538287553348863, 
0.00048642055896092, 0.000516432851497653, 0.000502933882398428, 
0.00048826607507247, 0.00050556957979958, 0.000489344715019492, 
0.000412651475080693, 0.000398998558856398, 0.00042811083852741, 
0.000420592783943452, 0.00040561174774836, 0.000390264320625555, 
0.000402351040877185, 0.000416416906389459, 0.000406800454869433, 
0.000436914133282049, 0.000442142284807156, 0.000429210158249497, 
0.000385914757362574, 0.000370335105150589, 0.000353304627933565, 
0.00036120300377744, 0.000377947403884487), c(0.00239753832296605, 
0.00149810321407759, 0.00145180659871048, 0.00146539544151048, 
0.00134826386510388, 0.00100878998807097, 0.000936914451841572, 
0.000809259544329687, 0.00091134693720946, 0.000799260633380906, 
0.000693721010205674, 0.000614836799469595, 0.000560998894856855, 
0.000444258494659727, 0.000449546260054794, 0.000378985501183941, 
0.000414331992454195, 0.000327825090424264, 0.000353925260488294, 
0.000401635605972863, 0.000353066580379476, 0.000366440227095464, 
0.000348742039394109, 0.000383099725847322, 0.000304883994273095, 
0.000304395072098668, 0.000264707007769475, 0.000200707772376352, 
0.000244941217454458, 0.000277255117457477, 0.000264379273017407, 
0.000256049526257224, 0.000228158365777488, 0.000202198318357024, 
0.00023937410075039, 0.000247054669886886, 0.000281759844582608, 
0.000256060852210361, 0.000246607157387898, 0.000253535323147763, 
0.000225570409911682, 0.000206736450621512, 0.000202263911352976, 
0.000157918460062725, 0.000139970032510461, 0.000153891162738769, 
0.000190337306237115, 0.000178874928346147, 0.000138221598137415, 
0.000134393042864206, 0.00017206347666422, 0.00018966815814774, 
0.000153192068716291, 0.000177838803652643, 0.000146556098640621, 
0.000182688505453927, 0.000174160873949313, 0.00018797340223593, 
0.000246924361030111, 0.000294463257588724, 0.000306835200015461, 
0.000305214498530033, 0.000337968045965656, 0.000337387255025373, 
0.000342226606233019, 0.000323973903402572, 0.000322282004089188, 
0.000292371612916264, 0.000269445820269462, 0.000248882220955965, 
0.000237522230775312, 0.000236329769355559, 0.000216712805608366, 
0.0002249330790598, 0.000233720687792251))), .Names = c("newcol", 
"newcol_std"))

EDIT:

I have the following plots and I am trying to plot the standard deviation (similar to the first picture).

The graphs are the the AUC score after each nround from CV. So basically the model improves less after the number of nrounds in xgb.cv

The graphs are not the ROC graphs as @missuse pointed out..

enter image description here

Improve this question
5
  • 1
    what are the values in the list? specifically newcol what does it represent? Commented Apr 7, 2018 at 10:31
  • newcol represents the AUC from a training set and newcol_std is the standard deviation of that AUC computed from an xgboost model. Commented Apr 7, 2018 at 10:32
  • 1
    To plot a roc curve you need either predicted probability and the true class, or sensitivity and specificity over a range of probabilities. auc is the are under the curve. What you say is that all your values are auc values, that would mean you expect 6 * 75 roc curves? But you did not provide data to plot them. Commented Apr 7, 2018 at 10:39
  • Damn, I completely messed up! After reading that artical and the way my results are shaped I lost concentration! What I meant to attempt was to plot the AUC (which is an average over k-fold CV) with the standard deviation, across the number of nrounds. I will update my original question now to make it more clear! Thanks for pointing out my mistake Commented Apr 7, 2018 at 10:44
  • 1
    No problem. I recommend deleting the first part of the question since it only adds confusion. Also I added answer, check please Commented Apr 7, 2018 at 10:54

1 Answer 1

Reset to default
2

Here is an approach:

library(tidyverse)

do.call(cbind, df$newcol) %>% #convert the first list element to a data frame
  as.data.frame %>%
  mutate(n = 1:n()) %>% #label observations 
  gather(key, value, 1:6) %>% #convert to long format
  bind_cols(do.call(cbind, df$newcol_std) %>% #convert the second list element to a data frame
                 as.data.frame %>%
                 gather(key, value)%>% #convert to long format
              select(-key)) %>% #drop unneeded column
  ggplot()+
    geom_line(aes(x = n, y = value, color = key))+
    geom_ribbon(aes(x = n, ymin = value-value1, ymax = value+value1, fill = key), alpha = 0.4)

enter image description here

Improve this answer
Sign up to request clarification or add additional context in comments.

2 Comments

Thanks! that was incredibly fast. One final question regarding these graphs. Does it make sense to you to plot these? I mean, to me they do but I am still relaively newish to all this. So, plotting the AUC result after each nround to see how the model performs is a good evaluation metric for CV? Are there any other methods I should be using to evaluate the model? I have the logloss and a few eval_metrics also and using a grid search for parameter optimisation.
To be honest I do not think it matters to see how any eval metric changed during the course of training. Every plot will be similar. I would say the only relevant thing is to compare eval metrics on the hold out samples (test auc in this case) at the end of training for various hyper parameters. There are much more important things you could plot, take a look at packages lime and xgboostExplainer

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.