1 /**************************************************************************
2 *
3 * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA
4 * All Rights Reserved.
5 * Copyright 2009 VMware, Inc., Palo Alto, CA., USA
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
23 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
24 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
25 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
26 * USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
28 **************************************************************************/
29 /*
30 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
31 */
32
33 #ifdef HAVE_CONFIG_H
34 #include "config.h"
35 #endif
36
37 #include "wsbm_fencemgr.h"
38 #include "wsbm_pool.h"
39 #include "wsbm_manager.h"
40 #include <xf86drm.h>
41 #include <drm/psb_ttm_fence_user.h>
42 #include <string.h>
43 #include <unistd.h>
44
45 struct _WsbmFenceClass
46 {
47 struct _WsbmListHead head;
48 struct _WsbmMutex mutex;
49 struct _WsbmMutex cmd_mutex;
50 };
51
52 /*
53 * Note: The struct _WsbmFenceMgr::Mutex should never be held
54 * during sleeps, since that may block fast concurrent access to
55 * fence data.
56 */
57
58 struct _WsbmFenceMgr
59 {
60 /*
61 * Constant members. Need no mutex protection.
62 */
63 struct _WsbmFenceMgrCreateInfo info;
64 void *private;
65
66 /*
67 * Atomic members. No mutex protection.
68 */
69
70 struct _WsbmAtomic count;
71
72 /*
73 * These members are protected by this->mutex
74 */
75
76 struct _WsbmFenceClass *classes;
77 uint32_t num_classes;
78 };
79
80 struct _WsbmFenceObject
81 {
82
83 /*
84 * These members are constant and need no mutex protection.
85 * Note that @private may point to a structure with its own
86 * mutex protection, that we don't care about.
87 */
88
89 struct _WsbmFenceMgr *mgr;
90 uint32_t fence_class;
91 uint32_t fence_type;
92 void *private;
93
94 /*
95 * Atomic members. No mutex protection. note that
96 * @signaled types is updated using a compare-and-swap
97 * scheme to guarantee atomicity.
98 */
99
100 struct _WsbmAtomic refCount;
101 struct _WsbmAtomic signaled_types;
102
103 /*
104 * These members are protected by mgr->mutex.
105 */
106 struct _WsbmListHead head;
107 };
108
109 uint32_t
wsbmFenceType(struct _WsbmFenceObject * fence)110 wsbmFenceType(struct _WsbmFenceObject *fence)
111 {
112 return fence->fence_type;
113 }
114
115 struct _WsbmFenceMgr *
wsbmFenceMgrCreate(const struct _WsbmFenceMgrCreateInfo * info)116 wsbmFenceMgrCreate(const struct _WsbmFenceMgrCreateInfo *info)
117 {
118 struct _WsbmFenceMgr *tmp;
119 uint32_t i, j;
120 int ret;
121
122 tmp = calloc(1, sizeof(*tmp));
123 if (!tmp)
124 return NULL;
125
126 tmp->info = *info;
127 tmp->classes = calloc(tmp->info.num_classes, sizeof(*tmp->classes));
128 if (!tmp->classes)
129 goto out_err;
130
131 for (i = 0; i < tmp->info.num_classes; ++i) {
132 struct _WsbmFenceClass *fc = &tmp->classes[i];
133
134 WSBMINITLISTHEAD(&fc->head);
135 ret = WSBM_MUTEX_INIT(&fc->mutex);
136 if (ret)
137 goto out_err1;
138 ret = WSBM_MUTEX_INIT(&fc->cmd_mutex);
139 if (ret) {
140 WSBM_MUTEX_FREE(&fc->mutex);
141 goto out_err1;
142 }
143 }
144 wsbmAtomicSet(&tmp->count, 0);
145
146 return tmp;
147
148 out_err1:
149 for (j = 0; j < i; ++j) {
150 WSBM_MUTEX_FREE(&tmp->classes[j].mutex);
151 WSBM_MUTEX_FREE(&tmp->classes[j].cmd_mutex);
152 }
153 free(tmp->classes);
154 out_err:
155 if (tmp)
156 free(tmp);
157 return NULL;
158 }
159
160 void
wsbmFenceUnreference(struct _WsbmFenceObject ** pFence)161 wsbmFenceUnreference(struct _WsbmFenceObject **pFence)
162 {
163 struct _WsbmFenceObject *fence = *pFence;
164 struct _WsbmFenceMgr *mgr;
165
166 *pFence = NULL;
167 if (fence == NULL)
168 return;
169
170 mgr = fence->mgr;
171 if (wsbmAtomicDecZero(&fence->refCount)) {
172 struct _WsbmFenceClass *fc = &mgr->classes[fence->fence_class];
173
174 WSBM_MUTEX_LOCK(&fc->mutex);
175 WSBMLISTDELINIT(&fence->head);
176 WSBM_MUTEX_UNLOCK(&fc->mutex);
177 if (fence->private)
178 mgr->info.unreference(mgr, &fence->private);
179 fence->mgr = NULL;
180 wsbmAtomicDecZero(&mgr->count);
181 free(fence);
182 }
183 }
184
185 static void
wsbmSignalPreviousFences(struct _WsbmFenceMgr * mgr,struct _WsbmListHead * list,uint32_t fence_class,uint32_t signaled_types)186 wsbmSignalPreviousFences(struct _WsbmFenceMgr *mgr,
187 struct _WsbmListHead *list,
188 uint32_t fence_class, uint32_t signaled_types)
189 {
190 struct _WsbmFenceClass *fc = &mgr->classes[fence_class];
191 struct _WsbmFenceObject *entry;
192 struct _WsbmListHead *prev;
193 uint32_t old_signaled_types;
194 uint32_t ret_st;
195
196 WSBM_MUTEX_LOCK(&fc->mutex);
197 while (list != &fc->head && list->next != list) {
198 entry = WSBMLISTENTRY(list, struct _WsbmFenceObject, head);
199
200 prev = list->prev;
201
202 do {
203 old_signaled_types = wsbmAtomicRead(&entry->signaled_types);
204 signaled_types =
205 old_signaled_types | (signaled_types & entry->fence_type);
206 if (signaled_types == old_signaled_types)
207 break;
208
209 ret_st =
210 wsbmAtomicCmpXchg(&entry->signaled_types, old_signaled_types,
211 signaled_types);
212 } while (ret_st != old_signaled_types);
213
214 if (signaled_types == entry->fence_type)
215 WSBMLISTDELINIT(list);
216
217 list = prev;
218 }
219 WSBM_MUTEX_UNLOCK(&fc->mutex);
220 }
221
222 int
wsbmFenceFinish(struct _WsbmFenceObject * fence,uint32_t fence_type,int lazy_hint)223 wsbmFenceFinish(struct _WsbmFenceObject *fence, uint32_t fence_type,
224 int lazy_hint)
225 {
226 struct _WsbmFenceMgr *mgr = fence->mgr;
227 int ret = 0;
228
229 if ((wsbmAtomicRead(&fence->signaled_types) & fence_type) == fence_type)
230 goto out;
231
232 ret = mgr->info.finish(mgr, fence->private, fence_type, lazy_hint);
233 if (ret)
234 goto out;
235
236 wsbmSignalPreviousFences(mgr, &fence->head, fence->fence_class,
237 fence_type);
238 out:
239 return ret;
240 }
241
242 uint32_t
wsbmFenceSignaledTypeCached(struct _WsbmFenceObject * fence)243 wsbmFenceSignaledTypeCached(struct _WsbmFenceObject * fence)
244 {
245 return wsbmAtomicRead(&fence->signaled_types);
246 }
247
248 int
wsbmFenceSignaledType(struct _WsbmFenceObject * fence,uint32_t flush_type,uint32_t * signaled)249 wsbmFenceSignaledType(struct _WsbmFenceObject *fence, uint32_t flush_type,
250 uint32_t * signaled)
251 {
252 int ret = 0;
253 struct _WsbmFenceMgr *mgr;
254 uint32_t signaled_types;
255 uint32_t old_signaled_types;
256 uint32_t ret_st;
257
258 mgr = fence->mgr;
259 *signaled = wsbmAtomicRead(&fence->signaled_types);
260 if ((*signaled & flush_type) == flush_type)
261 goto out0;
262
263 ret = mgr->info.signaled(mgr, fence->private, flush_type, signaled);
264 if (ret) {
265 *signaled = wsbmAtomicRead(&fence->signaled_types);
266 goto out0;
267 }
268
269 do {
270 old_signaled_types = wsbmAtomicRead(&fence->signaled_types);
271 signaled_types = old_signaled_types | *signaled;
272 if (signaled_types == old_signaled_types)
273 break;
274
275 ret_st = wsbmAtomicCmpXchg(&fence->signaled_types, old_signaled_types,
276 signaled_types);
277 if (old_signaled_types == ret_st)
278 wsbmSignalPreviousFences(mgr, &fence->head, fence->fence_class,
279 *signaled);
280 } while (old_signaled_types != ret_st);
281
282 return 0;
283 out0:
284 return ret;
285 }
286
287 struct _WsbmFenceObject *
wsbmFenceReference(struct _WsbmFenceObject * fence)288 wsbmFenceReference(struct _WsbmFenceObject *fence)
289 {
290 if (fence == NULL)
291 return NULL;
292 wsbmAtomicInc(&fence->refCount);
293 return fence;
294 }
295
296 struct _WsbmFenceObject *
wsbmFenceCreateSig(struct _WsbmFenceMgr * mgr,uint32_t fence_class,uint32_t fence_type,uint32_t signaled_types,void * private,size_t private_size)297 wsbmFenceCreateSig(struct _WsbmFenceMgr *mgr, uint32_t fence_class,
298 uint32_t fence_type, uint32_t signaled_types,
299 void *private, size_t private_size)
300 {
301 struct _WsbmFenceClass *fc = &mgr->classes[fence_class];
302 struct _WsbmFenceObject *fence;
303 size_t fence_size = sizeof(*fence);
304
305 if (private_size)
306 fence_size = ((fence_size + 15) & ~15);
307
308 fence = calloc(1, fence_size + private_size);
309
310 if (!fence)
311 goto out_err;
312
313 wsbmAtomicSet(&fence->refCount, 1);
314 fence->mgr = mgr;
315 fence->fence_class = fence_class;
316 fence->fence_type = fence_type;
317 wsbmAtomicSet(&fence->signaled_types, signaled_types);
318 fence->private = private;
319 if (private_size) {
320 fence->private = (void *)(((uint8_t *) fence) + fence_size);
321 memcpy(fence->private, private, private_size);
322 }
323
324 WSBM_MUTEX_LOCK(&fc->mutex);
325 WSBMLISTADDTAIL(&fence->head, &fc->head);
326 WSBM_MUTEX_UNLOCK(&fc->mutex);
327 wsbmAtomicInc(&mgr->count);
328 return fence;
329
330 out_err:
331 {
332 int ret = mgr->info.finish(mgr, private, fence_type, 0);
333
334 if (ret)
335 usleep(10000000);
336 }
337 if (fence)
338 free(fence);
339
340 mgr->info.unreference(mgr, &private);
341 return NULL;
342 }
343
344 struct _WsbmFenceObject *
wsbmFenceCreate(struct _WsbmFenceMgr * mgr,uint32_t fence_class,uint32_t fence_type,void * private,size_t private_size)345 wsbmFenceCreate(struct _WsbmFenceMgr *mgr, uint32_t fence_class,
346 uint32_t fence_type, void *private, size_t private_size)
347 {
348 return wsbmFenceCreateSig(mgr, fence_class, fence_type, 0, private,
349 private_size);
350 }
351
352 struct _WsbmTTMFenceMgrPriv
353 {
354 int fd;
355 unsigned int devOffset;
356 };
357
358 static int
tSignaled(struct _WsbmFenceMgr * mgr,void * private,uint32_t flush_type,uint32_t * signaled_type)359 tSignaled(struct _WsbmFenceMgr *mgr, void *private, uint32_t flush_type,
360 uint32_t * signaled_type)
361 {
362 struct _WsbmTTMFenceMgrPriv *priv =
363 (struct _WsbmTTMFenceMgrPriv *)mgr->private;
364 union ttm_fence_signaled_arg arg;
365 int ret;
366
367 arg.req.handle = (unsigned long)private;
368 arg.req.fence_type = flush_type;
369 arg.req.flush = 1;
370 *signaled_type = 0;
371
372 ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_SIGNALED,
373 &arg, sizeof(arg));
374 if (ret)
375 return ret;
376
377 *signaled_type = arg.rep.signaled_types;
378 return 0;
379 }
380
381 static int
tFinish(struct _WsbmFenceMgr * mgr,void * private,uint32_t fence_type,int lazy_hint)382 tFinish(struct _WsbmFenceMgr *mgr, void *private, uint32_t fence_type,
383 int lazy_hint)
384 {
385 struct _WsbmTTMFenceMgrPriv *priv =
386 (struct _WsbmTTMFenceMgrPriv *)mgr->private;
387 union ttm_fence_finish_arg arg =
388 {.req = {.handle = (unsigned long)private,
389 .fence_type = fence_type,
390 .mode = (lazy_hint) ? TTM_FENCE_FINISH_MODE_LAZY : 0}
391 };
392 int ret;
393
394 do {
395 ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_FINISH,
396 &arg, sizeof(arg));
397 } while (ret == -EAGAIN || ret == -ERESTART);
398
399 return ret;
400 }
401
402 static int
tUnref(struct _WsbmFenceMgr * mgr,void ** private)403 tUnref(struct _WsbmFenceMgr *mgr, void **private)
404 {
405 struct _WsbmTTMFenceMgrPriv *priv =
406 (struct _WsbmTTMFenceMgrPriv *)mgr->private;
407 struct ttm_fence_unref_arg arg = {.handle = (unsigned long)*private };
408
409 *private = NULL;
410
411 return drmCommandWrite(priv->fd, priv->devOffset + TTM_FENCE_UNREF,
412 &arg, sizeof(arg));
413 }
414
415 struct _WsbmFenceMgr *
wsbmFenceMgrTTMInit(int fd,unsigned int numClass,unsigned int devOffset)416 wsbmFenceMgrTTMInit(int fd, unsigned int numClass, unsigned int devOffset)
417 {
418 struct _WsbmFenceMgrCreateInfo info;
419 struct _WsbmFenceMgr *mgr;
420 struct _WsbmTTMFenceMgrPriv *priv = malloc(sizeof(*priv));
421
422 if (!priv)
423 return NULL;
424
425 priv->fd = fd;
426 priv->devOffset = devOffset;
427
428 info.flags = WSBM_FENCE_CLASS_ORDERED;
429 info.num_classes = numClass;
430 info.signaled = tSignaled;
431 info.finish = tFinish;
432 info.unreference = tUnref;
433
434 mgr = wsbmFenceMgrCreate(&info);
435 if (mgr == NULL) {
436 free(priv);
437 return NULL;
438 }
439
440 mgr->private = (void *)priv;
441 return mgr;
442 }
443
444 void
wsbmFenceCmdLock(struct _WsbmFenceMgr * mgr,uint32_t fence_class)445 wsbmFenceCmdLock(struct _WsbmFenceMgr *mgr, uint32_t fence_class)
446 {
447 WSBM_MUTEX_LOCK(&mgr->classes[fence_class].cmd_mutex);
448 }
449
450 void
wsbmFenceCmdUnlock(struct _WsbmFenceMgr * mgr,uint32_t fence_class)451 wsbmFenceCmdUnlock(struct _WsbmFenceMgr *mgr, uint32_t fence_class)
452 {
453 WSBM_MUTEX_UNLOCK(&mgr->classes[fence_class].cmd_mutex);
454 }
455
456 void
wsbmFenceMgrTTMTakedown(struct _WsbmFenceMgr * mgr)457 wsbmFenceMgrTTMTakedown(struct _WsbmFenceMgr *mgr)
458 {
459 unsigned int i;
460
461 if (!mgr)
462 return;
463
464 if (mgr->private)
465 free(mgr->private);
466
467 for (i = 0; i < mgr->info.num_classes; ++i) {
468 WSBM_MUTEX_FREE(&mgr->classes[i].mutex);
469 WSBM_MUTEX_FREE(&mgr->classes[i].cmd_mutex);
470 }
471 free(mgr);
472
473 return;
474 }
475